diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index c4aa43c775b1e..de0212685a8a7 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -11,7 +11,7 @@ "set_commit_status": false, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*(buildkite\\s*)?test\\s+this(\\s+please)?)", + "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)", "cancel_intermediate_builds": true, "cancel_intermediate_builds_on_comment": false }, diff --git a/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap b/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap index 6df8ca8b63438..50dea7a07e042 100644 --- a/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap +++ b/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap @@ -201,3 +201,111 @@ exports[`generatePipelines should generate correct pipeline when using a trigger }, ] `; + +exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticmachine 1`] = ` +[ + { + "name": "bwc-snapshots", + "pipeline": { + "steps": [ + { + "group": "bwc-snapshots", + "steps": [ + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest", + "env": { + "BWC_VERSION": "{{matrix.BWC_VERSION}}", + }, + "label": "{{matrix.BWC_VERSION}} / bwc-snapshots", + "matrix": { + "setup": { + "BWC_VERSION": [ + "7.17.14", + "8.10.3", + "8.11.0", + ], + }, + }, + "timeout_in_minutes": 300, + }, + ], + }, + ], + }, + }, + { + "name": "using-defaults", + "pipeline": { + "env": { + "CUSTOM_ENV_VAR": "value", + }, + "steps": [ + { + "command": "echo 'hello world'", + "label": "test-step", + }, + ], + }, + }, +] +`; + +exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticsearchmachine 1`] = ` +[ + { + "name": "bwc-snapshots", + "pipeline": { + "steps": [ + { + "group": "bwc-snapshots", + "steps": [ + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest", + "env": { + "BWC_VERSION": "{{matrix.BWC_VERSION}}", + }, + "label": "{{matrix.BWC_VERSION}} / bwc-snapshots", + "matrix": { + "setup": { + "BWC_VERSION": [ + "7.17.14", + "8.10.3", + "8.11.0", + ], + }, + }, + "timeout_in_minutes": 300, + }, + ], + }, + ], + }, + }, + { + "name": "using-defaults", + "pipeline": { + "env": { + "CUSTOM_ENV_VAR": "value", + }, + "steps": [ + { + "command": "echo 'hello world'", + "label": "test-step", + }, + ], + }, + }, +] +`; diff --git a/.buildkite/scripts/pull-request/pipeline.test.ts b/.buildkite/scripts/pull-request/pipeline.test.ts index d0634752260e4..562f37abbae1f 100644 --- a/.buildkite/scripts/pull-request/pipeline.test.ts +++ b/.buildkite/scripts/pull-request/pipeline.test.ts @@ -13,11 +13,11 @@ describe("generatePipelines", () => { }); // Helper for testing pipeline generations that should be the same when using the overall ci trigger comment "buildkite test this" - const testWithTriggerCheck = (directory: string, changedFiles?: string[]) => { + const testWithTriggerCheck = (directory: string, changedFiles?: string[], comment = "buildkite test this") => { const pipelines = generatePipelines(directory, changedFiles); expect(pipelines).toMatchSnapshot(); - process.env["GITHUB_PR_TRIGGER_COMMENT"] = "buildkite test this"; + process.env["GITHUB_PR_TRIGGER_COMMENT"] = comment; const pipelinesWithTriggerComment = generatePipelines(directory, changedFiles); expect(pipelinesWithTriggerComment).toEqual(pipelines); }; @@ -42,4 +42,20 @@ describe("generatePipelines", () => { const pipelines = generatePipelines(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]); expect(pipelines).toMatchSnapshot(); }); + + test("should generate correct pipelines with a non-docs change and @elasticmachine", () => { + testWithTriggerCheck( + `${import.meta.dir}/mocks/pipelines`, + ["build.gradle", "docs/README.asciidoc"], + "@elasticmachine test this please" + ); + }); + + test("should generate correct pipelines with a non-docs change and @elasticsearchmachine", () => { + testWithTriggerCheck( + `${import.meta.dir}/mocks/pipelines`, + ["build.gradle", "docs/README.asciidoc"], + "@elasticsearchmachine test this please" + ); + }); }); diff --git a/.buildkite/scripts/pull-request/pipeline.ts b/.buildkite/scripts/pull-request/pipeline.ts index 65aec47fe3cc8..6cb0e5d76b74b 100644 --- a/.buildkite/scripts/pull-request/pipeline.ts +++ b/.buildkite/scripts/pull-request/pipeline.ts @@ -148,7 +148,9 @@ export const generatePipelines = ( // However, if we're using the overall CI trigger "[buildkite] test this [please]", we should use the regular filters above if ( process.env["GITHUB_PR_TRIGGER_COMMENT"] && - !process.env["GITHUB_PR_TRIGGER_COMMENT"].match(/^\s*(buildkite\s*)?test\s+this(\s+please)?/i) + !process.env["GITHUB_PR_TRIGGER_COMMENT"].match( + /^\s*((@elastic(search)?machine|buildkite)\s*)?test\s+this(\s+please)?/i + ) ) { filters = [triggerCommentCheck]; } diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 934d9f05d77a2..758cdf687e6b6 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -143,6 +143,10 @@ gradlePlugin { id = 'elasticsearch.mrjar' implementationClass = 'org.elasticsearch.gradle.internal.MrjarPlugin' } + embeddedProvider { + id = 'elasticsearch.embedded-providers' + implementationClass = 'org.elasticsearch.gradle.internal.EmbeddedProviderPlugin' + } releaseTools { id = 'elasticsearch.release-tools' implementationClass = 'org.elasticsearch.gradle.internal.release.ReleaseToolsPlugin' diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java new file mode 100644 index 0000000000000..d3f79f7f76d4f --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.Directory; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.Sync; + +import static org.elasticsearch.gradle.internal.conventions.GUtils.capitalize; +import static org.elasticsearch.gradle.util.GradleUtils.getJavaSourceSets; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.DIRECTORY_TYPE; + +public class EmbeddedProviderExtension { + + private final Project project; + + public EmbeddedProviderExtension(Project project) { + this.project = project; + } + + void impl(String implName, Project implProject) { + String projectName = implProject.getName(); + String capitalName = capitalize(projectName); + + Configuration implConfig = project.getConfigurations().detachedConfiguration(project.getDependencies().create(implProject)); + implConfig.attributes(attrs -> { + attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_TYPE); + attrs.attribute(EmbeddedProviderPlugin.IMPL_ATTR, true); + }); + + String manifestTaskName = "generate" + capitalName + "ProviderManifest"; + Provider generatedResourcesDir = project.getLayout().getBuildDirectory().dir("generated-resources"); + var generateProviderManifest = project.getTasks().register(manifestTaskName, GenerateProviderManifest.class); + generateProviderManifest.configure(t -> { + t.getManifestFile().set(generatedResourcesDir.map(d -> d.file("LISTING.TXT"))); + t.getProviderImplClasspath().from(implConfig); + }); + + String implTaskName = "generate" + capitalName + "ProviderImpl"; + var generateProviderImpl = project.getTasks().register(implTaskName, Sync.class); + generateProviderImpl.configure(t -> { + t.into(generatedResourcesDir); + t.into("IMPL-JARS/" + implName, spec -> { + spec.from(implConfig); + spec.from(generateProviderManifest); + }); + }); + + var mainSourceSet = getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + mainSourceSet.getOutput().dir(generateProviderImpl); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderPlugin.java new file mode 100644 index 0000000000000..213730139d915 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderPlugin.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.elasticsearch.gradle.transform.UnzipTransform; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.attributes.Attribute; + +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.DIRECTORY_TYPE; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.JAR_TYPE; + +public class EmbeddedProviderPlugin implements Plugin { + static final Attribute IMPL_ATTR = Attribute.of("is.impl", Boolean.class); + + @Override + public void apply(Project project) { + + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom().attribute(ARTIFACT_TYPE_ATTRIBUTE, JAR_TYPE).attribute(IMPL_ATTR, true); + transformSpec.getTo().attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_TYPE).attribute(IMPL_ATTR, true); + transformSpec.parameters(parameters -> parameters.getIncludeArtifactName().set(true)); + }); + + project.getExtensions().create("embeddedProviders", EmbeddedProviderExtension.class, project); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java index d69a355a3595d..d03ec4ab09c95 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java @@ -98,6 +98,7 @@ public abstract class CheckForbiddenApisTask extends DefaultTask implements Patt private File resourcesDir; private boolean ignoreFailures = false; + private boolean ignoreMissingClasses = false; @Input @Optional @@ -250,6 +251,15 @@ public void setIgnoreFailures(boolean ignoreFailures) { this.ignoreFailures = ignoreFailures; } + @Input + public boolean getIgnoreMissingClasses() { + return ignoreMissingClasses; + } + + public void setIgnoreMissingClasses(boolean ignoreMissingClasses) { + this.ignoreMissingClasses = ignoreMissingClasses; + } + /** * The default compiler target version used to expand references to bundled JDK signatures. * E.g., if you use "jdk-deprecated", it will expand to this version. @@ -378,6 +388,7 @@ public void checkForbidden() { parameters.getSignatures().set(getSignatures()); parameters.getTargetCompatibility().set(getTargetCompatibility()); parameters.getIgnoreFailures().set(getIgnoreFailures()); + parameters.getIgnoreMissingClasses().set(getIgnoreMissingClasses()); parameters.getSuccessMarker().set(getSuccessMarker()); parameters.getSignaturesFiles().from(getSignaturesFiles()); }); @@ -514,7 +525,9 @@ private URLClassLoader createClassLoader(FileCollection classpath, FileCollectio @NotNull private Checker createChecker(URLClassLoader urlLoader) { final EnumSet options = EnumSet.noneOf(Checker.Option.class); - options.add(FAIL_ON_MISSING_CLASSES); + if (getParameters().getIgnoreMissingClasses().get() == false) { + options.add(FAIL_ON_MISSING_CLASSES); + } if (getParameters().getIgnoreFailures().get() == false) { options.add(FAIL_ON_VIOLATION); } @@ -573,6 +586,8 @@ interface Parameters extends WorkParameters { Property getIgnoreFailures(); + Property getIgnoreMissingClasses(); + ListProperty getSignatures(); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java index bddf95cae77d4..0270ee22ca8c5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java @@ -101,7 +101,7 @@ private AdoptiumVersionInfo toVersionInfo(JsonNode node) { private URI resolveDownloadURI(AdoptiumVersionRequest request, AdoptiumVersionInfo versionInfo) { return URI.create( "https://api.adoptium.net/v3/binary/version/jdk-" - + versionInfo.openjdkVersion + + versionInfo.semver + "/" + request.platform + "/" diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json index 87a9313f3eefe..2a0cda7fa33c9 100644 --- a/build-tools-internal/src/main/resources/changelog-schema.json +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -59,6 +59,7 @@ "Infra/Scripting", "Infra/Settings", "Infra/Transport API", + "Infra/Metrics", "Ingest", "Ingest Node", "Java High Level REST Client", diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy index 7b8129f8dbaec..6383d577f027f 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolverSpec.groovy @@ -42,7 +42,7 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { 1, 1, "" + languageVersion.asInt() + ".1.1.1+37", - 0, "" + languageVersion.asInt() + ".1.1.1" + 0, "" + languageVersion.asInt() + ".1.1.1+37.1" ))) } @@ -52,22 +52,22 @@ class AdoptiumJdkToolchainResolverSpec extends AbstractToolchainResolverSpec { @Override def supportedRequests() { return [ - [19, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [19, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [19, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [19, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], - [19, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], + [19, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [19, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [19, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [19, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], + [19, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-19.1.1.1+37.1/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], - [18, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [18, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [18, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [18, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], - [18, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], - [17, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [17, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [17, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"], - [17, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], - [17, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"] + [18, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [18, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [18, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [18, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], + [18, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-18.1.1.1+37.1/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], + [17, ADOPTIUM, MAC_OS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/mac/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [17, ADOPTIUM, LINUX, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/linux/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [17, ADOPTIUM, WINDOWS, X86_64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/windows/x64/jdk/hotspot/normal/eclipse?project=jdk"], + [17, ADOPTIUM, MAC_OS, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/mac/aarch64/jdk/hotspot/normal/eclipse?project=jdk"], + [17, ADOPTIUM, LINUX, AARCH64, "https://api.adoptium.net/v3/binary/version/jdk-17.1.1.1+37.1/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk"] ] } diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java index f587163b9324f..c25b422a980a3 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugin.noop.action.bulk.RestNoopBulkAction; import org.elasticsearch.plugin.noop.action.bulk.TransportNoopBulkAction; import org.elasticsearch.plugin.noop.action.search.RestNoopSearchAction; @@ -30,6 +31,7 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class NoopPlugin extends Plugin implements ActionPlugin { @@ -54,7 +56,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList(new RestNoopBulkAction(), new RestNoopSearchAction()); } diff --git a/docs/changelog/101209.yaml b/docs/changelog/101209.yaml new file mode 100644 index 0000000000000..debec27e61307 --- /dev/null +++ b/docs/changelog/101209.yaml @@ -0,0 +1,6 @@ +pr: 101209 +summary: "Making `k` and `num_candidates` optional for knn search" +area: Vector Search +type: enhancement +issues: + - 97533 diff --git a/docs/changelog/103481.yaml b/docs/changelog/103481.yaml new file mode 100644 index 0000000000000..f7c7c0b6eecc9 --- /dev/null +++ b/docs/changelog/103481.yaml @@ -0,0 +1,5 @@ +pr: 103481 +summary: Redirect failed ingest node operations to a failure store when available +area: Data streams +type: feature +issues: [] diff --git a/docs/changelog/103648.yaml b/docs/changelog/103648.yaml new file mode 100644 index 0000000000000..d4fa489a6812c --- /dev/null +++ b/docs/changelog/103648.yaml @@ -0,0 +1,5 @@ +pr: 103648 +summary: Introduce experimental pass-through field type +area: TSDB +type: enhancement +issues: [] diff --git a/docs/changelog/103973.yaml b/docs/changelog/103973.yaml new file mode 100644 index 0000000000000..f3bde76c7a559 --- /dev/null +++ b/docs/changelog/103973.yaml @@ -0,0 +1,5 @@ +pr: 103973 +summary: Add stricter validation for api key expiration time +area: Security +type: enhancement +issues: [] diff --git a/docs/changelog/104320.yaml b/docs/changelog/104320.yaml new file mode 100644 index 0000000000000..d2b0d09070fb9 --- /dev/null +++ b/docs/changelog/104320.yaml @@ -0,0 +1,5 @@ +pr: 104320 +summary: Hot-reloadable LDAP bind password +area: Authentication +type: enhancement +issues: [] diff --git a/docs/changelog/104363.yaml b/docs/changelog/104363.yaml new file mode 100644 index 0000000000000..9d97991ea7fab --- /dev/null +++ b/docs/changelog/104363.yaml @@ -0,0 +1,5 @@ +pr: 104363 +summary: Apply windowing and chunking to long documents +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/104529.yaml b/docs/changelog/104529.yaml new file mode 100644 index 0000000000000..5b223a0924d86 --- /dev/null +++ b/docs/changelog/104529.yaml @@ -0,0 +1,5 @@ +pr: 104529 +summary: Add rest spec for Query User API +area: Client +type: enhancement +issues: [] diff --git a/docs/changelog/104636.yaml b/docs/changelog/104636.yaml new file mode 100644 index 0000000000000..d74682f2eba18 --- /dev/null +++ b/docs/changelog/104636.yaml @@ -0,0 +1,5 @@ +pr: 104636 +summary: Modifying request builders +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/changelog/104648.yaml b/docs/changelog/104648.yaml new file mode 100644 index 0000000000000..e8bb5fea392ac --- /dev/null +++ b/docs/changelog/104648.yaml @@ -0,0 +1,5 @@ +pr: 104648 +summary: "[Connector API] Implement update `index_name` action" +area: Application +type: enhancement +issues: [] diff --git a/docs/changelog/104665.yaml b/docs/changelog/104665.yaml new file mode 100644 index 0000000000000..a7043cbdc9dda --- /dev/null +++ b/docs/changelog/104665.yaml @@ -0,0 +1,5 @@ +pr: 104665 +summary: Restrict usage of certain aggregations when in sort order execution is required +area: TSDB +type: enhancement +issues: [] diff --git a/docs/changelog/104750.yaml b/docs/changelog/104750.yaml new file mode 100644 index 0000000000000..948b19a5eaaa6 --- /dev/null +++ b/docs/changelog/104750.yaml @@ -0,0 +1,5 @@ +pr: 104750 +summary: "[Connectors API] Implement connector status update action" +area: Application +type: enhancement +issues: [] diff --git a/docs/changelog/104872.yaml b/docs/changelog/104872.yaml new file mode 100644 index 0000000000000..ad70946be02ae --- /dev/null +++ b/docs/changelog/104872.yaml @@ -0,0 +1,5 @@ +pr: 104872 +summary: Add new int8_flat and flat vector index types +area: Vector Search +type: enhancement +issues: [] diff --git a/docs/changelog/104878.yaml b/docs/changelog/104878.yaml new file mode 100644 index 0000000000000..2ae6d5c0c1da3 --- /dev/null +++ b/docs/changelog/104878.yaml @@ -0,0 +1,5 @@ +pr: 104878 +summary: "Transforms: Adding basic stats API param" +area: Transform +type: enhancement +issues: [] diff --git a/docs/changelog/104982.yaml b/docs/changelog/104982.yaml new file mode 100644 index 0000000000000..62194aa68b80c --- /dev/null +++ b/docs/changelog/104982.yaml @@ -0,0 +1,5 @@ +pr: 104982 +summary: "[Connectors API] Add new field `api_key_secret_id` to Connector" +area: Application +type: enhancement +issues: [] diff --git a/docs/changelog/104993.yaml b/docs/changelog/104993.yaml new file mode 100644 index 0000000000000..df9875563d5a1 --- /dev/null +++ b/docs/changelog/104993.yaml @@ -0,0 +1,5 @@ +pr: 104993 +summary: Support enrich remote mode +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/104996.yaml b/docs/changelog/104996.yaml new file mode 100644 index 0000000000000..b94711111adfe --- /dev/null +++ b/docs/changelog/104996.yaml @@ -0,0 +1,5 @@ +pr: 104996 +summary: "Enhancement: Metrics for Search Took Times using Action Listeners" +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/105015.yaml b/docs/changelog/105015.yaml new file mode 100644 index 0000000000000..94ffc2b0e58d5 --- /dev/null +++ b/docs/changelog/105015.yaml @@ -0,0 +1,5 @@ +pr: 105015 +summary: Modify name of threadpool metric for rejected +area: Infra/Metrics +type: enhancement +issues: [] diff --git a/docs/changelog/105024.yaml b/docs/changelog/105024.yaml new file mode 100644 index 0000000000000..96268b78ddf5d --- /dev/null +++ b/docs/changelog/105024.yaml @@ -0,0 +1,6 @@ +pr: 105024 +summary: "[Connectors API] Fix bug with crawler configuration parsing and `sync_now`\ + \ flag" +area: Application +type: bug +issues: [] diff --git a/docs/changelog/105044.yaml b/docs/changelog/105044.yaml new file mode 100644 index 0000000000000..5a9a11f928f98 --- /dev/null +++ b/docs/changelog/105044.yaml @@ -0,0 +1,5 @@ +pr: 105044 +summary: Expose `OperationPurpose` via `CustomQueryParameter` to s3 logs +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/105055.yaml b/docs/changelog/105055.yaml new file mode 100644 index 0000000000000..0db70a6b9e558 --- /dev/null +++ b/docs/changelog/105055.yaml @@ -0,0 +1,5 @@ +pr: 105055 +summary: "Do not enable APM agent 'instrument', it's not required for manual tracing" +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/105062.yaml b/docs/changelog/105062.yaml new file mode 100644 index 0000000000000..928786f62381a --- /dev/null +++ b/docs/changelog/105062.yaml @@ -0,0 +1,5 @@ +pr: 105062 +summary: Nest pass-through objects within objects +area: TSDB +type: enhancement +issues: [] diff --git a/docs/changelog/105066.yaml b/docs/changelog/105066.yaml new file mode 100644 index 0000000000000..95757a9edaf81 --- /dev/null +++ b/docs/changelog/105066.yaml @@ -0,0 +1,5 @@ +pr: 105066 +summary: Fix handling of `ml.config_version` node attribute for nodes with machine learning disabled +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/105070.yaml b/docs/changelog/105070.yaml new file mode 100644 index 0000000000000..ff4c115e21eea --- /dev/null +++ b/docs/changelog/105070.yaml @@ -0,0 +1,5 @@ +pr: 105070 +summary: Validate settings before reloading JWT shared secret +area: Authentication +type: bug +issues: [] diff --git a/docs/changelog/105089.yaml b/docs/changelog/105089.yaml new file mode 100644 index 0000000000000..6f43c58af8a41 --- /dev/null +++ b/docs/changelog/105089.yaml @@ -0,0 +1,6 @@ +pr: 105089 +summary: Return results in order +area: Transform +type: bug +issues: + - 104847 diff --git a/docs/changelog/105131.yaml b/docs/changelog/105131.yaml new file mode 100644 index 0000000000000..36993527da583 --- /dev/null +++ b/docs/changelog/105131.yaml @@ -0,0 +1,5 @@ +pr: 105131 +summary: "[Connector API] Support filtering by name, index name in list action" +area: Application +type: enhancement +issues: [] diff --git a/docs/changelog/105153.yaml b/docs/changelog/105153.yaml new file mode 100644 index 0000000000000..6c6b1f995df4b --- /dev/null +++ b/docs/changelog/105153.yaml @@ -0,0 +1,6 @@ +pr: 105153 +summary: Field-caps should read fields from up-to-dated shards +area: "Search" +type: bug +issues: + - 104809 diff --git a/docs/changelog/96235.yaml b/docs/changelog/96235.yaml new file mode 100644 index 0000000000000..83d1eaf74916b --- /dev/null +++ b/docs/changelog/96235.yaml @@ -0,0 +1,5 @@ +pr: 96235 +summary: Add `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting to ignore dynamic fields when field limit is reached +area: Mapping +type: enhancement +issues: [] diff --git a/docs/changelog/99142.yaml b/docs/changelog/99142.yaml new file mode 100644 index 0000000000000..885946cec909b --- /dev/null +++ b/docs/changelog/99142.yaml @@ -0,0 +1,6 @@ +pr: 99142 +summary: Reuse number field mapper tests in other modules +area: Search +type: enhancement +issues: + - 92947 diff --git a/docs/reference/aggregations/bucket/time-series-aggregation.asciidoc b/docs/reference/aggregations/bucket/time-series-aggregation.asciidoc index d93df55118a8b..1fb527cd645f0 100644 --- a/docs/reference/aggregations/bucket/time-series-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/time-series-aggregation.asciidoc @@ -97,3 +97,14 @@ further. Alternatively, using sub aggregations can limit the amount of values re The `keyed` parameter determines if buckets are returned as a map with unique keys per bucket. By default with `keyed` set to false, buckets are returned as an array. + +[[times-series-aggregations-limitations]] +==== Limitations + +The `time_series` aggregation has many limitations. Many aggregation performance optimizations are disabled when using +the `time_series` aggregation. For example the filter by filter optimization or collect mode breath first (`terms` and +`multi_terms` aggregation forcefully use the depth first collect mode). + +The following aggregations also fail to work if used in combination with the `time_series` aggregation: +`auto_date_histogram`, `variable_width_histogram`, `rare_terms`, `global`, `composite`, `sampler`, `random_sampler` and +`diversified_sampler`. diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc index 1f0a79258bd37..25d39a17af306 100644 --- a/docs/reference/indices/flush.asciidoc +++ b/docs/reference/indices/flush.asciidoc @@ -81,7 +81,7 @@ Defaults to `open`. If `true`, the request forces a flush even if there are no changes to commit to the index. -Defaults to `true`. +Defaults to `false`. You can use this parameter to increment the generation number of the transaction log. diff --git a/docs/reference/mapping/fields/ignored-field.asciidoc b/docs/reference/mapping/fields/ignored-field.asciidoc index 5249d2d379a8e..5fd6c478438ab 100644 --- a/docs/reference/mapping/fields/ignored-field.asciidoc +++ b/docs/reference/mapping/fields/ignored-field.asciidoc @@ -4,8 +4,11 @@ The `_ignored` field indexes and stores the names of every field in a document that has been ignored when the document was indexed. This can, for example, be the case when the field was malformed and <> -was turned on, or when a `keyword` fields value exceeds its optional -<> setting. +was turned on, when a `keyword` field's value exceeds its optional +<> setting, or when +<> has been reached and +<> +is set to `true`. This field is searchable with <>, <> and <> diff --git a/docs/reference/mapping/mapping-settings-limit.asciidoc b/docs/reference/mapping/mapping-settings-limit.asciidoc index c499ca7675f2c..6e05e6ea60855 100644 --- a/docs/reference/mapping/mapping-settings-limit.asciidoc +++ b/docs/reference/mapping/mapping-settings-limit.asciidoc @@ -20,9 +20,18 @@ limits the maximum number of clauses in a query. + [TIP] ==== -If your field mappings contain a large, arbitrary set of keys, consider using the <> data type. +If your field mappings contain a large, arbitrary set of keys, consider using the <> data type, +or setting the index setting `index.mapping.total_fields.ignore_dynamic_beyond_limit` to `true`. ==== +`index.mapping.total_fields.ignore_dynamic_beyond_limit`:: + This setting determines what happens when a dynamically mapped field would exceed the total fields limit. + When set to `false` (the default), the index request of the document that tries to add a dynamic field to the mapping will fail with the message `Limit of total fields [X] has been exceeded`. + When set to `true`, the index request will not fail. + Instead, fields that would exceed the limit are not added to the mapping, similar to <>. + The fields that were not added to the mapping will be added to the <>. + The default value is `false`. + `index.mapping.depth.limit`:: The maximum depth for a field, which is measured as the number of inner objects. For instance, if all fields are defined at the root object level, diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc index ac95d5de80b89..094828fb445dd 100644 --- a/docs/reference/mapping/params/dynamic.asciidoc +++ b/docs/reference/mapping/params/dynamic.asciidoc @@ -90,3 +90,10 @@ accepts the following parameters: to the mapping, and new fields must be added explicitly. `strict`:: If new fields are detected, an exception is thrown and the document is rejected. New fields must be explicitly added to the mapping. + +[[dynamic-field-limit]] +==== Behavior when reaching the field limit +Setting `dynamic` to either `true` or `runtime` will only add dynamic fields until <> is reached. +By default, index requests for documents that would exceed the field limit will fail, +unless <> is set to `true`. +In that case, ignored fields are added to the <>. diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index a2ab44a173a62..d600bc5566ace 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -238,21 +238,31 @@ expense of slower indexing speed. ==== `type`::: (Required, string) -The type of kNN algorithm to use. Can be either `hnsw` or `int8_hnsw`. - +The type of kNN algorithm to use. Can be either any of: ++ +-- +* `hnsw` - The default storage type. This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] for scalable + approximate kNN search. This supports all `element_type` values. +* `int8_hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically scalar +quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint +by 4x at the cost of some accuracy. See <>. +* `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values. +* `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports +`element_type` of `float`. +-- `m`::: (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. -Defaults to `16`. +Defaults to `16`. Only applicable to `hnsw` and `int8_hnsw` index types. `ef_construction`::: (Optional, integer) The number of candidates to track while assembling the list of nearest -neighbors for each new node. Defaults to `100`. +neighbors for each new node. Defaults to `100`. Only applicable to `hnsw` and `int8_hnsw` index types. `confidence_interval`::: (Optional, float) -Only applicable to `int8_hnsw` index types. The confidence interval to use when quantizing the vectors, +Only applicable to `int8_hnsw` and `int8_flat` index types. The confidence interval to use when quantizing the vectors, can be any value between and including `0.90` and `1.0`. This value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc index 5a7aa43155c66..a73a3906bd3fd 100644 --- a/docs/reference/modules/cluster/shards_allocation.asciidoc +++ b/docs/reference/modules/cluster/shards_allocation.asciidoc @@ -22,37 +22,54 @@ one of the active allocation ids in the cluster state. -- +[[cluster-routing-allocation-same-shard-host]] +`cluster.routing.allocation.same_shard.host`:: + (<>) + If `true`, forbids multiple copies of a shard from being allocated to + distinct nodes on the same host, i.e. which have the same network + address. Defaults to `false`, meaning that copies of a shard may + sometimes be allocated to nodes on the same host. This setting is only + relevant if you run multiple nodes on each host. + `cluster.routing.allocation.node_concurrent_incoming_recoveries`:: (<>) - How many concurrent incoming shard recoveries are allowed to happen on a node. Incoming recoveries are the recoveries - where the target shard (most likely the replica unless a shard is relocating) is allocated on the node. Defaults to `2`. + How many concurrent incoming shard recoveries are allowed to happen on a + node. Incoming recoveries are the recoveries where the target shard (most + likely the replica unless a shard is relocating) is allocated on the node. + Defaults to `2`. Increasing this setting may cause shard movements to have + a performance impact on other activity in your cluster, but may not make + shard movements complete noticeably sooner. We do not recommend adjusting + this setting from its default of `2`. `cluster.routing.allocation.node_concurrent_outgoing_recoveries`:: (<>) - How many concurrent outgoing shard recoveries are allowed to happen on a node. Outgoing recoveries are the recoveries - where the source shard (most likely the primary unless a shard is relocating) is allocated on the node. Defaults to `2`. + How many concurrent outgoing shard recoveries are allowed to happen on a + node. Outgoing recoveries are the recoveries where the source shard (most + likely the primary unless a shard is relocating) is allocated on the node. + Defaults to `2`. Increasing this setting may cause shard movements to have + a performance impact on other activity in your cluster, but may not make + shard movements complete noticeably sooner. We do not recommend adjusting + this setting from its default of `2`. `cluster.routing.allocation.node_concurrent_recoveries`:: (<>) - A shortcut to set both `cluster.routing.allocation.node_concurrent_incoming_recoveries` and - `cluster.routing.allocation.node_concurrent_outgoing_recoveries`. Defaults to 2. - + A shortcut to set both + `cluster.routing.allocation.node_concurrent_incoming_recoveries` and + `cluster.routing.allocation.node_concurrent_outgoing_recoveries`. Defaults + to `2`. Increasing this setting may cause shard movements to have a + performance impact on other activity in your cluster, but may not make + shard movements complete noticeably sooner. We do not recommend adjusting + this setting from its default of `2`. `cluster.routing.allocation.node_initial_primaries_recoveries`:: - (<>) - While the recovery of replicas happens over the network, the recovery of - an unassigned primary after node restart uses data from the local disk. - These should be fast so more initial primary recoveries can happen in - parallel on the same node. Defaults to `4`. - -[[cluster-routing-allocation-same-shard-host]] -`cluster.routing.allocation.same_shard.host`:: - (<>) - If `true`, forbids multiple copies of a shard from being allocated to - distinct nodes on the same host, i.e. which have the same network - address. Defaults to `false`, meaning that copies of a shard may - sometimes be allocated to nodes on the same host. This setting is only - relevant if you run multiple nodes on each host. + (<>) + While the recovery of replicas happens over the network, the recovery of + an unassigned primary after node restart uses data from the local disk. + These should be fast so more initial primary recoveries can happen in + parallel on each node. Defaults to `4`. Increasing this setting may cause + shard recoveries to have a performance impact on other activity in your + cluster, but may not make shard recoveries complete noticeably sooner. We + do not recommend adjusting this setting from its default of `4`. [[shards-rebalancing-settings]] ==== Shard rebalancing settings @@ -73,38 +90,44 @@ balancer works independently within each tier. You can use the following settings to control the rebalancing of shards across the cluster: -`cluster.routing.rebalance.enable`:: +`cluster.routing.allocation.allow_rebalance`:: + -- (<>) -Enable or disable rebalancing for specific kinds of shards: +Specify when shard rebalancing is allowed: -* `all` - (default) Allows shard balancing for all kinds of shards. -* `primaries` - Allows shard balancing only for primary shards. -* `replicas` - Allows shard balancing only for replica shards. -* `none` - No shard balancing of any kind are allowed for any indices. + +* `always` - Always allow rebalancing. +* `indices_primaries_active` - Only when all primaries in the cluster are allocated. +* `indices_all_active` - (default) Only when all shards (primaries and replicas) in the cluster are allocated. -- -`cluster.routing.allocation.allow_rebalance`:: +`cluster.routing.rebalance.enable`:: + -- (<>) -Specify when shard rebalancing is allowed: +Enable or disable rebalancing for specific kinds of shards: +* `all` - (default) Allows shard balancing for all kinds of shards. +* `primaries` - Allows shard balancing only for primary shards. +* `replicas` - Allows shard balancing only for replica shards. +* `none` - No shard balancing of any kind are allowed for any indices. -* `always` - Always allow rebalancing. -* `indices_primaries_active` - Only when all primaries in the cluster are allocated. -* `indices_all_active` - (default) Only when all shards (primaries and replicas) in the cluster are allocated. +Rebalancing is important to ensure the cluster returns to a healthy and fully +resilient state after a disruption. If you adjust this setting, remember to set +it back to `all` as soon as possible. -- `cluster.routing.allocation.cluster_concurrent_rebalance`:: (<>) Defines the number of concurrent shard rebalances are allowed across the whole cluster. Defaults to `2`. Note that this setting only controls the number of -concurrent shard relocations due to imbalances in the cluster. This setting does -not limit shard relocations due to +concurrent shard relocations due to imbalances in the cluster. This setting +does not limit shard relocations due to <> or -<>. +<>. Increasing this setting may cause the +cluster to use additional resources moving shards between nodes, so we +generally do not recommend adjusting this setting from its default of `2`. `cluster.routing.allocation.type`:: + @@ -149,6 +172,12 @@ data stream have an estimated write load of zero. The following settings control how {es} combines these values into an overall measure of each node's weight. +`cluster.routing.allocation.balance.threshold`:: +(float, <>) +The minimum improvement in weight which triggers a rebalancing shard movement. +Defaults to `1.0f`. Raising this value will cause {es} to stop rebalancing +shards sooner, leaving the cluster in a more unbalanced state. + `cluster.routing.allocation.balance.shard`:: (float, <>) Defines the weight factor for the total number of shards allocated to each node. @@ -177,19 +206,25 @@ estimated number of indexing threads needed by the shard. Defaults to `10.0f`. Raising this value increases the tendency of {es} to equalize the total write load across nodes ahead of the other balancing variables. -`cluster.routing.allocation.balance.threshold`:: -(float, <>) -The minimum improvement in weight which triggers a rebalancing shard movement. -Defaults to `1.0f`. Raising this value will cause {es} to stop rebalancing -shards sooner, leaving the cluster in a more unbalanced state. - [NOTE] ==== -* It is not recommended to adjust the values of the heuristics settings. The -default values are generally good, and although different values may improve -the current balance, it is possible that they create problems in the future -if the cluster or workload changes. +* If you have a large cluster, it may be unnecessary to keep it in +a perfectly balanced state at all times. It is less resource-intensive for the +cluster to operate in a somewhat unbalanced state rather than to perform all +the shard movements needed to achieve the perfect balance. If so, increase the +value of `cluster.routing.allocation.balance.threshold` to define the +acceptable imbalance between nodes. For instance, if you have an average of 500 +shards per node and can accept a difference of 5% (25 typical shards) between +nodes, set `cluster.routing.allocation.balance.threshold` to `25`. + +* We do not recommend adjusting the values of the heuristic weight factor +settings. The default values work well in all reasonable clusters. Although +different values may improve the current balance in some ways, it is possible +that they will create unexpected problems in the future or prevent it from +gracefully handling an unexpected disruption. + * Regardless of the result of the balancing algorithm, rebalancing might not be allowed due to allocation rules such as forced awareness and allocation -filtering. +filtering. Use the <> API to explain the current +allocation of shards. ==== diff --git a/docs/reference/modules/indices/recovery.asciidoc b/docs/reference/modules/indices/recovery.asciidoc index 02b70c69876ff..261c3d3fc3f24 100644 --- a/docs/reference/modules/indices/recovery.asciidoc +++ b/docs/reference/modules/indices/recovery.asciidoc @@ -38,8 +38,9 @@ This limit applies to each node separately. If multiple nodes in a cluster perform recoveries at the same time, the cluster's total recovery traffic may exceed this limit. + -If this limit is too high, ongoing recoveries may consume an excess of bandwidth -and other resources, which can destabilize the cluster. +If this limit is too high, ongoing recoveries may consume an excess of +bandwidth and other resources, which can have a performance impact on your +cluster and in extreme cases may destabilize it. + This is a dynamic setting, which means you can set it in each node's `elasticsearch.yml` config file and you can update it dynamically using the diff --git a/docs/reference/query-dsl/knn-query.asciidoc b/docs/reference/query-dsl/knn-query.asciidoc index 1f4297cb4e089..e9aeea68c06f7 100644 --- a/docs/reference/query-dsl/knn-query.asciidoc +++ b/docs/reference/query-dsl/knn-query.asciidoc @@ -94,10 +94,10 @@ as the vector field you are searching against. `num_candidates`:: + -- -(Required, integer) The number of nearest neighbor candidates to consider per shard. +(Optional, integer) The number of nearest neighbor candidates to consider per shard. Cannot exceed 10,000. {es} collects `num_candidates` results from each shard, then merges them to find the top results. Increasing `num_candidates` tends to improve the -accuracy of the final results. +accuracy of the final results. Defaults to `Math.min(1.5 * size, 10_000)`. -- `filter`:: diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index ca8a191ad4b2c..a5ab70ae85181 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -584,14 +584,15 @@ end::knn-filter[] tag::knn-k[] Number of nearest neighbors to return as top hits. This value must be less than -`num_candidates`. +`num_candidates`. Defaults to `size`. end::knn-k[] tag::knn-num-candidates[] -The number of nearest neighbor candidates to consider per shard. Cannot exceed -10,000. {es} collects `num_candidates` results from each shard, then merges them +The number of nearest neighbor candidates to consider per shard. +Needs to be greater than `k`, or `size` if `k` is omitted, and cannot exceed 10,000. +{es} collects `num_candidates` results from each shard, then merges them to find the top `k` results. Increasing `num_candidates` tends to improve the -accuracy of the final `k` results. +accuracy of the final `k` results. Defaults to `Math.min(1.5 * k, 10_000)`. end::knn-num-candidates[] tag::knn-query-vector[] diff --git a/docs/reference/search/knn-search.asciidoc b/docs/reference/search/knn-search.asciidoc index b8522bccb916d..136b53388baf9 100644 --- a/docs/reference/search/knn-search.asciidoc +++ b/docs/reference/search/knn-search.asciidoc @@ -102,22 +102,22 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] `knn`:: -(Required, object) +(Required, object) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn] + .Properties of `knn` object [%collapsible%open] ==== `field`:: -(Required, string) +(Required, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field] `k`:: -(Required, integer) +(Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k] `num_candidates`:: -(Required, integer) +(Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates] `query_vector`:: diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 96e3ae43f98dc..cd7d496d89d7b 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -505,11 +505,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] `k`:: -(Required, integer) +(Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k] `num_candidates`:: -(Required, integer) +(Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates] `query_vector`:: diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc index 858902bb72ef2..49501c46b8ba9 100644 --- a/docs/reference/setup/install.asciidoc +++ b/docs/reference/setup/install.asciidoc @@ -16,8 +16,8 @@ To set up Elasticsearch in {ecloud}, sign up for a {ess-trial}[free {ecloud} tri If you want to install and manage {es} yourself, you can: -* Run {es} on any Linux, MacOS, or Windows machine. -* Run {es} in a <>. +* Run {es} using a <>. +* Run {es} in a <>. * Set up and manage {es}, {kib}, {agent}, and the rest of the Elastic Stack on Kubernetes with {eck-ref}[{eck}]. TIP: To try out Elasticsearch on your own machine, we recommend using Docker and running both Elasticsearch and Kibana. For more information, see <>. @@ -57,10 +57,18 @@ Elasticsearch website or from our RPM repository. + <> +TIP: For a step-by-step example of setting up the {stack} on your own premises, try out our tutorial: {stack-ref}/installing-stack-demo-self.html[Installing a self-managed Elastic Stack]. + +[discrete] +[[elasticsearch-docker-images]] +=== Elasticsearch container images + +You can also run {es} inside a container image. + +[horizontal] `docker`:: -Images are available for running Elasticsearch as Docker containers. They may be -downloaded from the Elastic Docker Registry. +Docker container images may be downloaded from the Elastic Docker Registry. + {ref}/docker.html[Install {es} with Docker] diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc index 8dfbca8c63210..a30c8c313b263 100644 --- a/docs/reference/setup/install/rpm.asciidoc +++ b/docs/reference/setup/install/rpm.asciidoc @@ -19,6 +19,8 @@ NOTE: Elasticsearch includes a bundled version of https://openjdk.java.net[OpenJ from the JDK maintainers (GPLv2+CE). To use your own version of Java, see the <> +TIP: For a step-by-step example of setting up the {stack} on your own premises, try out our tutorial: {stack-ref}/installing-stack-demo-self.html[Installing a self-managed Elastic Stack]. + [[rpm-key]] ==== Import the Elasticsearch GPG Key diff --git a/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc b/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc index 48e9f802e13f8..5ba18df3e6a6b 100644 --- a/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc +++ b/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc @@ -41,13 +41,16 @@ timing out in the browser's Developer Tools Network tab. doesn't normally cause problems unless it's combined with overriding <>. The default `1000` limit is considered generous, though overriding to `10000` -doesn't cause noticable impact depending on use case. However, to give +doesn't cause noticeable impact depending on use case. However, to give a bad example, overriding to `100000` and this limit being hit by mapping totals would usually have strong performance implications. If your index mapped fields expect to contain a large, arbitrary set of keys, you may instead consider: +* Setting <> to `true`. +Instead of rejecting documents that exceed the field limit, this will ignore dynamic fields once the limit is reached. + * Using the <> data type. Please note, however, that flattened objects is link:https://github.com/elastic/kibana/issues/25820[not fully supported in {kib}] yet. For example, this could apply to sub-mappings like { `host.name` , `host.os`, `host.version` }. Desired fields are still accessed by diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle index 5c9dd49c007b8..15a79364559a2 100644 --- a/libs/x-content/build.gradle +++ b/libs/x-content/build.gradle @@ -6,44 +6,17 @@ * Side Public License, v 1. */ - -import org.elasticsearch.gradle.transform.UnzipTransform -import org.elasticsearch.gradle.internal.GenerateProviderManifest -import org.gradle.api.internal.artifacts.ArtifactAttributes - -import java.util.stream.Collectors - apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' +apply plugin: 'elasticsearch.embedded-providers' -def isImplAttr = Attribute.of("is.impl", Boolean) - -configurations { - providerImpl { - attributes.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE) - attributes.attribute(isImplAttr, true) - } +embeddedProviders { + impl 'x-content', project(':libs:elasticsearch-x-content:impl') } dependencies { - registerTransform( - UnzipTransform.class, transformSpec -> { - transformSpec.getFrom() - .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.JAR_TYPE) - .attribute(isImplAttr, true) - transformSpec.getTo() - .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE) - .attribute(isImplAttr, true) - transformSpec.parameters(parameters -> { - parameters.includeArtifactName.set(true) - }) - - }) - api project(':libs:elasticsearch-core') - providerImpl project(':libs:elasticsearch-x-content:impl') - testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content' } @@ -66,18 +39,3 @@ tasks.named("thirdPartyAudit").configure { tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' } - -Directory generatedResourcesDir = layout.buildDirectory.dir('generated-resources').get() -def generateProviderManifest = tasks.register("generateProviderManifest", GenerateProviderManifest.class) { - manifestFile = generatedResourcesDir.file("LISTING.TXT") - getProviderImplClasspath().from(configurations.providerImpl) -} - -def generateProviderImpl = tasks.register("generateProviderImpl", Sync) { - destinationDir = generatedResourcesDir.dir("impl").getAsFile() - into("IMPL-JARS/x-content") { - from(configurations.providerImpl) - from(generateProviderManifest) - } -} -sourceSets.main.output.dir(generateProviderImpl) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index 917d8f0b80f2c..3ff2e618f5759 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -41,11 +41,13 @@ import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -76,7 +78,7 @@ public class TimeSeriesAggregationsIT extends AggregationIntegTestCase { @Override public void setupSuiteScopeCluster() throws Exception { - int numberOfIndices = randomIntBetween(1, 3); + int numberOfIndices = randomIntBetween(1, 1); numberOfDimensions = randomIntBetween(1, 5); numberOfMetrics = randomIntBetween(1, 10); String[] routingKeys = randomSubsetOf( @@ -146,7 +148,7 @@ public void setupSuiteScopeCluster() throws Exception { for (int i = 0; i < numberOfDocs; i++) { XContentBuilder docSource = XContentFactory.jsonBuilder(); docSource.startObject(); - Map key = new HashMap<>(); + Map key = new TreeMap<>(Comparator.naturalOrder()); for (int d = 0; d < numberOfDimensions; d++) { String dim = randomFrom(dimensions[d]); docSource.field("dim_" + d, dim); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java new file mode 100644 index 0000000000000..18b24123e6cf0 --- /dev/null +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.aggregations.bucket; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.aggregations.AggregationIntegTestCase; +import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; +import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalCardinality; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.junit.Before; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Supplier; + +public class TimeSeriesAggregationsUnlimitedDimensionsIT extends AggregationIntegTestCase { + private static int numberOfDimensions; + private static int numberOfDocuments; + + @Before + public void setup() throws Exception { + numberOfDimensions = randomIntBetween(25, 99); + final XContentBuilder mapping = timeSeriesIndexMapping(); + long startMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + long endMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-02T00:00:00Z"); + numberOfDocuments = randomIntBetween(100, 200); + final Iterator timestamps = getTimestamps(startMillis, endMillis, numberOfDocuments); + // NOTE: use also the last (changing) dimension so to make sure documents are not indexed all in the same shard. + final String[] routingDimensions = new String[] { "dim_0", "dim_" + (numberOfDimensions - 1) }; + assertTrue(prepareTimeSeriesIndex(mapping, startMillis, endMillis, routingDimensions).isAcknowledged()); + + logger.info("Dimensions: " + numberOfDimensions + " docs: " + numberOfDocuments + " start: " + startMillis + " end: " + endMillis); + + // NOTE: we need the tsid to be larger than 32 Kb + final String fooDimValue = "foo_" + randomAlphaOfLengthBetween(2048, 2048 + 128); + final String barDimValue = "bar_" + randomAlphaOfLengthBetween(2048, 2048 + 256); + final String bazDimValue = "baz_" + randomAlphaOfLengthBetween(2048, 2048 + 512); + + final BulkRequestBuilder bulkIndexRequest = client().prepareBulk(); + for (int docId = 0; docId < numberOfDocuments; docId++) { + final XContentBuilder document = timeSeriesDocument(fooDimValue, barDimValue, bazDimValue, docId, timestamps::next); + bulkIndexRequest.add(client().prepareIndex("index").setOpType(DocWriteRequest.OpType.CREATE).setSource(document)); + } + BulkResponse bulkIndexResponse = bulkIndexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + assertFalse(bulkIndexResponse.hasFailures()); + assertEquals(RestStatus.OK.getStatus(), client().admin().indices().prepareFlush("index").get().getStatus().getStatus()); + } + + private static XContentBuilder timeSeriesDocument( + final String fooDimValue, + final String barDimValue, + final String bazDimValue, + int docId, + final Supplier timestampSupplier + ) throws IOException { + final XContentBuilder docSource = XContentFactory.jsonBuilder(); + docSource.startObject(); + // NOTE: we assign dimensions in such a way that almost all of them have the same value but the last one. + // This way we are going to have just two time series (and two distinct tsid) and the last dimension identifies + // which time series the document belongs to. + for (int dimId = 0; dimId < numberOfDimensions - 1; dimId++) { + docSource.field("dim_" + dimId, fooDimValue); + } + docSource.field("dim_" + (numberOfDimensions - 1), docId % 2 == 0 ? barDimValue : bazDimValue); + docSource.field("counter_metric", docId + 1); + docSource.field("gauge_metric", randomDoubleBetween(1000.0, 2000.0, true)); + docSource.field("@timestamp", timestampSupplier.get()); + docSource.endObject(); + + return docSource; + } + + private CreateIndexResponse prepareTimeSeriesIndex( + final XContentBuilder mapping, + long startMillis, + long endMillis, + final String[] routingDimensions + ) { + return prepareCreate("index").setSettings( + Settings.builder() + .put("mode", "time_series") + .put("routing_path", String.join(",", routingDimensions)) + .put("index.number_of_shards", randomIntBetween(1, 3)) + .put("index.number_of_replicas", randomIntBetween(1, 3)) + .put("time_series.start_time", startMillis) + .put("time_series.end_time", endMillis) + .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), 4192) + .build() + ).setMapping(mapping).get(); + } + + private static Iterator getTimestamps(long startMillis, long endMillis, int numberOfDocs) { + final Set timestamps = new TreeSet<>(); + while (timestamps.size() < numberOfDocs) { + timestamps.add(randomLongBetween(startMillis, endMillis)); + } + return timestamps.iterator(); + } + + private static XContentBuilder timeSeriesIndexMapping() throws IOException { + final XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("properties"); + for (int i = 0; i < numberOfDimensions; i++) { + builder.startObject("dim_" + i); + builder.field("type", "keyword"); + builder.field("time_series_dimension", true); + builder.endObject(); + } + builder.startObject("counter_metric"); + builder.field("type", "double"); + builder.field("time_series_metric", "counter"); + builder.endObject(); + builder.startObject("gauge_metric"); + builder.field("type", "double"); + builder.field("time_series_metric", "gauge"); + builder.endObject(); + builder.endObject(); // properties + builder.endObject(); + return builder; + } + + public void testTimeSeriesAggregation() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts"); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + aggregationResponse.decRef(); + } + } + + public void testSumByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new SumAggregationBuilder("sum").field("gauge_metric") + ); + final SearchResponse searchResponse = client().prepareSearch("index").setQuery(new MatchAllQueryBuilder()).get(); + assertNotEquals(numberOfDocuments, searchResponse.getHits().getHits().length); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + searchResponse.decRef(); + aggregationResponse.decRef(); + } + } + + public void testTermsByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new TermsAggregationBuilder("terms").field("dim_0") + ); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + aggregationResponse.decRef(); + } + } + + public void testDateHistogramByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new DateHistogramAggregationBuilder("date_histogram").field("@timestamp").calendarInterval(DateHistogramInterval.MINUTE) + ); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + aggregationResponse.decRef(); + } + } + + public void testCardinalityByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new CardinalityAggregationBuilder("dim_n_cardinality").field("dim_" + (numberOfDimensions - 1)) + ); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + try { + ((InternalTimeSeries) aggregationResponse.getAggregations().get("ts")).getBuckets().forEach(bucket -> { + assertCardinality(bucket.getAggregations().get("dim_n_cardinality"), 1); + }); + } finally { + aggregationResponse.decRef(); + } + } + + private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { + final var dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); + // NOTE: only two time series expected as a result of having just two distinct values for the last dimension + assertEquals(2, dimensions.size()); + + final Object firstTimeSeries = dimensions.get(0); + final Object secondTimeSeries = dimensions.get(1); + + assertNotEquals(firstTimeSeries, secondTimeSeries); + } + + private static void assertCardinality(final InternalCardinality cardinalityAggregation, int expectedCardinality) { + assertEquals(expectedCardinality, cardinalityAggregation.getValue()); + } +} diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java index 5c58b7f7bff5a..3287f50ab1739 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java @@ -29,14 +29,11 @@ import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.hamcrest.Matchers; import org.junit.Before; import java.io.IOException; import java.util.Iterator; -import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.function.Supplier; @@ -113,7 +110,6 @@ private CreateIndexResponse prepareTimeSeriesIndex( .put("index.number_of_replicas", randomIntBetween(1, 3)) .put("time_series.start_time", startMillis) .put("time_series.end_time", endMillis) - .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), numberOfDimensions + 1) .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), 4192) .build() ).setMapping(mapping).get(); @@ -209,27 +205,14 @@ public void testCardinalityByTsid() { } private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { - final List> dimensions = timeSeriesAggregation.getBuckets() - .stream() - .map(InternalTimeSeries.InternalBucket::getKey) - .toList(); + final var dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); // NOTE: only two time series expected as a result of having just two distinct values for the last dimension assertEquals(2, dimensions.size()); - final Map firstTimeSeries = dimensions.get(0); - final Map secondTimeSeries = dimensions.get(1); + final Object firstTimeSeries = dimensions.get(0); + final Object secondTimeSeries = dimensions.get(1); - assertTsid(firstTimeSeries); - assertTsid(secondTimeSeries); - } - - private static void assertTsid(final Map timeSeries) { - timeSeries.entrySet().stream().sorted(Map.Entry.comparingByKey()).limit(numberOfDimensions - 2).forEach(entry -> { - assertThat(entry.getValue().toString(), Matchers.equalTo(FOO_DIM_VALUE)); - }); - timeSeries.entrySet().stream().sorted(Map.Entry.comparingByKey()).skip(numberOfDimensions - 1).forEach(entry -> { - assertThat(entry.getValue().toString(), Matchers.oneOf(BAR_DIM_VALUE, BAZ_DIM_VALUE)); - }); + assertNotEquals(firstTimeSeries, secondTimeSeries); } private static void assertCardinality(final InternalCardinality cardinalityAggregation, int expectedCardinality) { diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java new file mode 100644 index 0000000000000..97c75689fe5dc --- /dev/null +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -0,0 +1,298 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.aggregations.bucket; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; +import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; + +public class TimeSeriesTsidHashCardinalityIT extends ESSingleNodeTestCase { + private static final String START_TIME = "2021-01-01T00:00:00Z"; + private static final String END_TIME = "2021-12-31T23:59:59Z"; + private String beforeIndex, afterIndex; + private long startTime, endTime; + private int numDimensions, numTimeSeries; + + @Override + protected Collection> getPlugins() { + List> plugins = new ArrayList<>(super.getPlugins()); + plugins.add(InternalSettingsPlugin.class); + plugins.add(AggregationsPlugin.class); + return plugins; + } + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + beforeIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + afterIndex = randomAlphaOfLength(12).toLowerCase(Locale.ROOT); + startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(START_TIME); + endTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(END_TIME); + numTimeSeries = 500; + // NOTE: we need to use few dimensions to be able to index documents in an index created before introducing TSID hashing + numDimensions = randomIntBetween(10, 20); + + final Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("dim_routing")) + .put( + IndexSettings.TIME_SERIES_START_TIME.getKey(), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(Instant.ofEpochMilli(startTime).toEpochMilli()) + ) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), END_TIME); + + final XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties"); + mapping.startObject("@timestamp").field("type", "date").endObject(); + + // Dimensions + mapping.startObject("dim_routing").field("type", "keyword").field("time_series_dimension", true).endObject(); + for (int i = 1; i <= numDimensions; i++) { + mapping.startObject("dim_" + i).field("type", "keyword").field("time_series_dimension", true).endObject(); + } + // Metrics + mapping.startObject("gauge").field("type", "double").field("time_series_metric", "gauge").endObject(); + mapping.endObject().endObject().endObject(); + assertAcked( + indicesAdmin().prepareCreate(beforeIndex) + .setSettings( + settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.NEW_INDEXVERSION_FORMAT).build() + ) + .setMapping(mapping) + .get() + ); + + assertAcked( + indicesAdmin().prepareCreate(afterIndex) + .setSettings( + settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.TIME_SERIES_ID_HASHING).build() + ) + .setMapping(mapping) + .get() + ); + + final TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(); + while (timeSeriesDataset.size() < numTimeSeries) { + final Set dimensions = new TreeSet<>(); + for (int j = 0; j < numDimensions; j++) { + if (randomIntBetween(1, 10) >= 2) { // NOTE: 20% chance the dimension field is empty + dimensions.add(new Dimension("dim_" + j, randomAlphaOfLength(12))); + } + } + final TimeSeries ts = new TimeSeries(dimensions); + if (timeSeriesDataset.exists(ts)) { + continue; + } + for (int k = 0; k < randomIntBetween(10, 20); k++) { + ts.addValue(randomLongBetween(startTime, endTime), randomDoubleBetween(100.0D, 200.0D, true)); + } + timeSeriesDataset.add(ts); + } + + final BulkRequestBuilder beforeBulkIndexRequest = client().prepareBulk(); + final BulkRequestBuilder afterBulkIndexRequest = client().prepareBulk(); + for (final TimeSeries ts : timeSeriesDataset) { + for (final TimeSeriesValue timeSeriesValue : ts.values) { + final XContentBuilder docSource = XContentFactory.jsonBuilder(); + docSource.startObject(); + docSource.field("dim_routing", "foo"); // Just to make sure we have at least one routing dimension + for (int d = 0; d < numDimensions; d++) { + final Dimension dimension = ts.getDimension("dim_" + d); + if (dimension != null) { + docSource.field("dim_" + d, dimension.value); + } + } + docSource.field("@timestamp", timeSeriesValue.timestamp); + docSource.field("gauge", timeSeriesValue.gauge); + docSource.endObject(); + beforeBulkIndexRequest.add(prepareIndex(beforeIndex).setOpType(DocWriteRequest.OpType.CREATE).setSource(docSource)); + afterBulkIndexRequest.add(prepareIndex(afterIndex).setOpType(DocWriteRequest.OpType.CREATE).setSource(docSource)); + } + } + assertFalse(beforeBulkIndexRequest.get().hasFailures()); + assertFalse(afterBulkIndexRequest.get().hasFailures()); + assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(beforeIndex, afterIndex).get().getStatus()); + } + + public void testTimeSeriesNumberOfBuckets() { + final SearchResponse searchBefore = client().prepareSearch(beforeIndex) + .setSize(0) + .addAggregation(new TimeSeriesAggregationBuilder("ts")) + .get(); + final SearchResponse searchAfter = client().prepareSearch(afterIndex) + .setSize(0) + .addAggregation(new TimeSeriesAggregationBuilder("ts")) + .get(); + try { + final InternalTimeSeries beforeTimeSeries = searchBefore.getAggregations().get("ts"); + final InternalTimeSeries afterTimeSeries = searchAfter.getAggregations().get("ts"); + assertEquals(beforeTimeSeries.getBuckets().size(), afterTimeSeries.getBuckets().size()); + } finally { + searchBefore.decRef(); + searchAfter.decRef(); + } + } + + record Dimension(String name, String value) implements Comparable { + + @Override + public String toString() { + return "Dimension{" + "name='" + name + '\'' + ", value='" + value + '\'' + '}'; + } + + @Override + public int compareTo(final Dimension that) { + return Comparator.comparing(Dimension::name).thenComparing(Dimension::value).compare(this, that); + } + } + + record TimeSeriesValue(long timestamp, double gauge) { + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeSeriesValue that = (TimeSeriesValue) o; + return timestamp == that.timestamp && Double.compare(gauge, that.gauge) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, gauge); + } + + @Override + public String toString() { + return "TimeSeriesValue{" + "timestamp=" + timestamp + ", gauge=" + gauge + '}'; + } + } + + static class TimeSeries { + private final HashMap dimensions; + private final Set values; + + TimeSeries(final Set dimensions) { + this.dimensions = new HashMap<>(); + for (final Dimension dimension : dimensions) { + this.dimensions.put(dimension.name, dimension); + } + values = new TreeSet<>(Comparator.comparing(TimeSeriesValue::timestamp)); + } + + public void addValue(long timestamp, double gauge) { + values.add(new TimeSeriesValue(timestamp, gauge)); + } + + public String id() { + final StringBuilder sb = new StringBuilder(); + for (final Dimension dimension : dimensions.values().stream().sorted(Comparator.comparing(Dimension::name)).toList()) { + sb.append(dimension.name()).append("=").append(dimension.value()); + } + + return sb.toString(); + } + + public Dimension getDimension(final String name) { + return this.dimensions.get(name); + } + } + + static class TimeSeriesDataset implements Iterable { + private final HashMap dataset; + + TimeSeriesDataset() { + this.dataset = new HashMap<>(); + } + + public void add(final TimeSeries ts) { + TimeSeries previous = dataset.put(ts.id(), ts); + if (previous != null) { + throw new IllegalArgumentException("Existing time series: " + ts.id()); + } + } + + public boolean exists(final TimeSeries ts) { + return dataset.containsKey(ts.id()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeSeriesDataset that = (TimeSeriesDataset) o; + return Objects.equals(dataset, that.dataset); + } + + @Override + public int hashCode() { + return Objects.hash(dataset); + } + + @Override + public String toString() { + return "TimeSeriesDataset{" + "dataset=" + dataset + '}'; + } + + @Override + public Iterator iterator() { + return new TimeSeriesIterator(this.dataset.entrySet().iterator()); + } + + public int size() { + return this.dataset.size(); + } + + record TimeSeriesIterator(Iterator> it) implements Iterator { + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public TimeSeries next() { + return it.next().getValue(); + } + } + } +} diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java index ba838e9b4046f..67a7773fd01bb 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java @@ -66,7 +66,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Map getKey() { - return TimeSeriesIdFieldMapper.decodeTsid(key); + return TimeSeriesIdFieldMapper.decodeTsidAsMap(key); } @Override diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index d5b796f3b93c5..9cd7f7a86e532 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -8,8 +8,11 @@ package org.elasticsearch.aggregations.bucket.timeseries; +import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.Releasables; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.search.aggregations.AggregationExecutionContext; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -20,11 +23,16 @@ import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.BytesKeyedBucketOrds; import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; public class TimeSeriesAggregator extends BucketsAggregator { @@ -32,6 +40,8 @@ public class TimeSeriesAggregator extends BucketsAggregator { private final boolean keyed; private final int size; + private final SortedMap dimensionValueSources; + @SuppressWarnings("this-escape") public TimeSeriesAggregator( String name, @@ -47,6 +57,11 @@ public TimeSeriesAggregator( this.keyed = keyed; bucketOrds = BytesKeyedBucketOrds.build(bigArrays(), bucketCardinality); this.size = size; + dimensionValueSources = new TreeMap<>(); + for (var dim : context.subSearchContext().getSearchExecutionContext().dimensionFields()) { + var valueSource = ValuesSourceConfig.resolve(context, null, dim.name(), null, null, null, null, null).getValuesSource(); + dimensionValueSources.put(dim.name(), valueSource); + } } @Override @@ -56,14 +71,11 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]); List buckets = new ArrayList<>(); - BytesRef prev = null; while (ordsEnum.next()) { long docCount = bucketDocCount(ordsEnum.ord()); ordsEnum.readValue(spare); - assert prev == null || spare.compareTo(prev) > 0 - : "key [" + spare.utf8ToString() + "] is smaller than previous key [" + prev.utf8ToString() + "]"; InternalTimeSeries.InternalBucket bucket = new InternalTimeSeries.InternalBucket( - prev = BytesRef.deepCopyOf(spare), // Closing bucketOrds will corrupt the bytes ref, so need to make a deep copy here. + BytesRef.deepCopyOf(spare), // Closing bucketOrds will corrupt the bytes ref, so need to make a deep copy here. docCount, null, keyed @@ -74,6 +86,12 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I break; } } + // NOTE: after introducing _tsid hashing time series are sorted by (_tsid hash, @timestamp) instead of (_tsid, timestamp). + // _tsid hash and _tsid might sort differently, and out of order data might result in incorrect buckets due to _tsid value + // changes not matching _tsid hash changes. Changes in _tsid hash are handled creating a new bucket as a result of making + // the assumption that sorting data results in new buckets whenever there is a change in _tsid hash. This is no true anymore + // because we collect data sorted on (_tsid hash, timestamp) but build aggregation results sorted by (_tsid, timestamp). + buckets.sort(Comparator.comparing(bucket -> bucket.key)); allBucketsPerOrd[ordIdx] = buckets.toArray(new InternalTimeSeries.InternalBucket[0]); } buildSubAggsForAllBuckets(allBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a); @@ -97,6 +115,29 @@ protected void doClose() { @Override protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException { + SortedMap dimensionConsumers = new TreeMap<>(); + for (var entry : dimensionValueSources.entrySet()) { + String fieldName = entry.getKey(); + if (entry.getValue() instanceof ValuesSource.Numeric numericVS) { + SortedNumericDocValues docValues = numericVS.longValues(aggCtx.getLeafReaderContext()); + dimensionConsumers.put(entry.getKey(), (docId, tsidBuilder) -> { + if (docValues.advanceExact(docId)) { + for (int i = 0; i < docValues.docValueCount(); i++) { + tsidBuilder.addLong(fieldName, docValues.nextValue()); + } + } + }); + } else { + SortedBinaryDocValues docValues = entry.getValue().bytesValues(aggCtx.getLeafReaderContext()); + dimensionConsumers.put(entry.getKey(), (docId, tsidBuilder) -> { + if (docValues.advanceExact(docId)) { + for (int i = 0; i < docValues.docValueCount(); i++) { + tsidBuilder.addString(fieldName, docValues.nextValue()); + } + } + }); + } + } return new LeafBucketCollectorBase(sub, null) { // Keeping track of these fields helps to reduce time spent attempting to add bucket + tsid combos that already were added. @@ -111,12 +152,18 @@ public void collect(int doc, long bucket) throws IOException { // changes to what is stored in currentTsidOrd then that ordinal well never occur again. Same applies // currentBucket if there is no parent aggregation or the immediate parent aggregation creates buckets // based on @timestamp field or dimension fields (fields that make up the tsid). - if (currentBucket == bucket && currentTsidOrd == aggCtx.getTsidOrd()) { + if (currentBucket == bucket && currentTsidOrd == aggCtx.getTsidHashOrd()) { collectExistingBucket(sub, doc, currentBucketOrdinal); return; } - long bucketOrdinal = bucketOrds.add(bucket, aggCtx.getTsid()); + TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + for (TsidConsumer consumer : dimensionConsumers.values()) { + consumer.accept(doc, tsidBuilder); + } + + BytesRef tsid = tsidBuilder.buildLegacyTsid().toBytesRef(); + long bucketOrdinal = bucketOrds.add(bucket, tsid); if (bucketOrdinal < 0) { // already seen bucketOrdinal = -1 - bucketOrdinal; collectExistingBucket(sub, doc, bucketOrdinal); @@ -125,13 +172,19 @@ public void collect(int doc, long bucket) throws IOException { } currentBucketOrdinal = bucketOrdinal; - currentTsidOrd = aggCtx.getTsidOrd(); + currentTsidOrd = aggCtx.getTsidHashOrd(); currentBucket = bucket; } + }; } InternalTimeSeries buildResult(InternalTimeSeries.InternalBucket[] topBuckets) { return new InternalTimeSeries(name, List.of(topBuckets), keyed, metadata()); } + + @FunctionalInterface + interface TsidConsumer { + void accept(int docId, TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder) throws IOException; + } } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index cc3813e7ec53a..4a8b2c98aef14 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -45,7 +45,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a builder.addString(entry.getKey(), (String) entry.getValue()); } try { - var key = builder.build().toBytesRef(); + var key = builder.buildLegacyTsid().toBytesRef(); bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 880d223442e29..54d1c14931d92 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -14,16 +14,20 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.aggregations.bucket.AggregationTestCase; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder; @@ -74,8 +78,12 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { assertThat(((Sum) ts.getBucketByKey("{dim1=bbb, dim2=zzz}").getAggregations().get("sum")).value(), equalTo(22.0)); }, - new KeywordFieldMapper.KeywordFieldType("dim1"), - new KeywordFieldMapper.KeywordFieldType("dim2"), + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), + new KeywordFieldMapper.Builder("dim2", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) ); } @@ -88,8 +96,16 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { builder.addLong(dimensions[i].toString(), n.longValue()); + if (dimensions[i + 1] instanceof Integer || dimensions[i + 1] instanceof Long) { + fields.add(new NumericDocValuesField(dimensions[i].toString(), ((Number) dimensions[i + 1]).longValue())); + } else if (dimensions[i + 1] instanceof Float) { + fields.add(new FloatDocValuesField(dimensions[i].toString(), (float) dimensions[i + 1])); + } else if (dimensions[i + 1] instanceof Double) { + fields.add(new DoubleDocValuesField(dimensions[i].toString(), (double) dimensions[i + 1])); + } } else { builder.addString(dimensions[i].toString(), dimensions[i + 1].toString()); + fields.add(new SortedSetDocValuesField(dimensions[i].toString(), new BytesRef(dimensions[i + 1].toString()))); } } for (int i = 0; i < metrics.length; i += 2) { @@ -101,8 +117,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); } } - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.build().toBytesRef())); - // TODO: Handle metrics + fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.buildLegacyTsid().toBytesRef())); iw.addDocument(fields); } @@ -131,7 +146,9 @@ public void testWithDateHistogramExecutedAsFilterByFilterWithTimeSeriesIndexSear aggregationBuilder, TimeSeriesIdFieldMapper.FIELD_TYPE, new DateFieldMapper.DateFieldType("@timestamp"), - new KeywordFieldMapper.KeywordFieldType("dim1"), + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) ).withQuery(new MatchAllDocsQuery()) ); @@ -177,7 +194,18 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { dateBuilder.fixedInterval(DateHistogramInterval.seconds(1)); TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); tsBuilder.subAggregation(dateBuilder); - timeSeriesTestCase(tsBuilder, new MatchAllDocsQuery(), buildIndex, verifier); + timeSeriesTestCase( + tsBuilder, + new MatchAllDocsQuery(), + buildIndex, + verifier, + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), + new KeywordFieldMapper.Builder("dim2", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType() + ); } public void testAggregationSize() throws IOException { @@ -201,7 +229,18 @@ public void testAggregationSize() throws IOException { TimeSeriesAggregationBuilder limitedTsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); limitedTsBuilder.setSize(i); - timeSeriesTestCase(limitedTsBuilder, new MatchAllDocsQuery(), buildIndex, limitedVerifier); + timeSeriesTestCase( + limitedTsBuilder, + new MatchAllDocsQuery(), + buildIndex, + limitedVerifier, + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), + new KeywordFieldMapper.Builder("dim2", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType() + ); } } diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 33ca0dc4779b6..9e8ec6b3f6768 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -107,8 +107,8 @@ setup: --- "Size test": - skip: - version: " - 8.6.99" - reason: Size added in 8.7.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } - do: search: @@ -311,8 +311,8 @@ setup: --- "Number for keyword routing field": - skip: - version: " - 8.10.99" - reason: "Fix in 8.11" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: bulk: @@ -343,3 +343,170 @@ setup: - match: { aggregations.ts.buckets.0.key: { "key": "10" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } + - match: { aggregations.ts.buckets.1.key: { "key": "11" } } + - match: { aggregations.ts.buckets.1.doc_count: 1 } + +--- +"Multiple indices _tsid vs _tsid hash sorting": + # sort(_tsid, timestamp) != sort(_tsid hash, timestamp) might result in incorrect buckets in the aggregation result. + # Here dimension values are crafted in such a way that sorting on _tsid and sorting on _tsid hash results in different + # collection and reduction order. Note that changing the hashing algorithm might require selecting proper values + # for dimensions fields such that sort(_tsid, timestamp) != sort(_tsid hash, timestamp). + + - skip: + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 + + - do: + indices.create: + index: test-1 + body: + settings: + mode: time_series + routing_path: [ key ] + time_series: + start_time: "2021-04-01T00:00:00Z" + end_time: "2021-04-30T23:59:59Z" + number_of_shards: 1 + mappings: + properties: + key: + type: keyword + time_series_dimension: true + "@timestamp": + type: date + gauge: + type: double + time_series_metric: "gauge" + + - do: + indices.create: + index: test-2 + body: + settings: + mode: time_series + routing_path: [ key ] + time_series: + start_time: "2021-05-01T00:00:00Z" + end_time: "2022-05-31T23:59:59Z" + number_of_shards: 1 + mappings: + properties: + key: + type: keyword + time_series_dimension: true + "@timestamp": + type: date + gauge: + type: double + time_series_metric: "gauge" + + - do: + bulk: + index: test-1 + refresh: true + body: + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 1, "@timestamp": "2021-04-01T01:00:11Z" }' + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 2, "@timestamp": "2021-04-01T02:00:12Z" }' + - '{ "index": {} }' + - '{ "key": "foo", "gauge": 3, "@timestamp": "2021-04-01T03:00:13Z" }' + + - is_false: errors + + - do: + bulk: + index: test-2 + refresh: true + body: + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 10, "@timestamp": "2021-05-01T01:00:31Z" }' + - '{ "index": {} }' + - '{ "key": "foo", "gauge": 20, "@timestamp": "2021-05-01T02:00:32Z" }' + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 30, "@timestamp": "2021-05-01T03:00:33Z" }' + - '{ "index": {} }' + - '{ "key": "foo", "gauge": 40, "@timestamp": "2021-05-01T04:00:34Z" }' + + - is_false: errors + + - do: + search: + index: test-1,test-2 + body: + size: 0 + aggs: + ts: + time_series: + keyed: false + + - match: { hits.total.value: 7 } + - length: { aggregations: 1 } + + - length: { aggregations.ts.buckets: 2 } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.doc_count: 4 } + - match: { aggregations.ts.buckets.1.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.1.doc_count: 3 } + +--- +"auto_date_histogram aggregation with time_series aggregation": + - skip: + version: " - 8.12.99" + reason: "Handling for time series aggregation failures introduced in 8.13.0" + + - do: + catch: '/\[by_time\] aggregation is incompatible with time series execution mode/' + search: + index: tsdb + body: + aggs: + by_time: + auto_date_histogram: + field: "@timestamp" + aggs: + ts: + time_series: {} + +--- +"variable_width_histogram aggregation with time_series aggregation": + - skip: + version: " - 8.12.99" + reason: "Handling for time series aggregation failures introduced in 8.13.0" + + - do: + catch: '/\[variable_width_histogram\] aggregation is incompatible with time series execution mode/' + search: + index: tsdb + body: + aggs: + variable_width_histogram: + variable_width_histogram: + field: val + aggs: + ts: + time_series: {} + +--- +"rare_terms aggregation with time_series aggregation": + - skip: + version: " - 8.12.99" + reason: "Handling for time series aggregation failures introduced in 8.13.0" + + - do: + catch: '/\[rare_terms\] aggregation is incompatible with time series execution mode/' + search: + index: tsdb + body: + aggs: + ts: + time_series: {} + aggs: + rare_terms: + rare_terms: + field: key + aggs: + max: + max: + field: val diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java index 88359d32a628c..0bbaca00d1e2e 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java @@ -44,7 +44,6 @@ public void addClusterSettingsListeners(ClusterService clusterService, APMTeleme clusterSettings.addSettingsUpdateConsumer(TELEMETRY_TRACING_ENABLED_SETTING, enabled -> { apmTracer.setEnabled(enabled); - this.setAgentSetting("instrument", Boolean.toString(enabled)); // The agent records data other than spans, e.g. JVM metrics, so we toggle this setting in order to // minimise its impact to a running Elasticsearch. boolean recording = enabled || clusterSettings.get(TELEMETRY_METRICS_ENABLED_SETTING); @@ -73,7 +72,6 @@ public void initAgentSystemProperties(Settings settings) { boolean metrics = TELEMETRY_METRICS_ENABLED_SETTING.get(settings); this.setAgentSetting("recording", Boolean.toString(tracing || metrics)); - this.setAgentSetting("instrument", Boolean.toString(tracing)); // Apply values from the settings in the cluster state APM_AGENT_SETTINGS.getAsMap(settings).forEach(this::setAgentSetting); } @@ -120,7 +118,8 @@ public void setAgentSetting(String key, String value) { // Core: // forbid 'enabled', must remain enabled to dynamically enable tracing / metrics - // forbid 'recording' / 'instrument', controlled by 'telemetry.metrics.enabled' / 'telemetry.tracing.enabled' + // forbid 'recording', controlled by 'telemetry.metrics.enabled' / 'telemetry.tracing.enabled' + // forbid 'instrument', automatic instrumentation can cause issues "service_name", "service_node_name", // forbid 'service_version', forced by APMJvmOptions diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java index d7ae93aded3de..f075f4fc39cfd 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java @@ -60,13 +60,11 @@ public void testEnableTracing() { apmAgentSettings.initAgentSystemProperties(update); verify(apmAgentSettings).setAgentSetting("recording", "true"); - verify(apmAgentSettings).setAgentSetting("instrument", "true"); clearInvocations(apmAgentSettings); Settings initial = Settings.builder().put(update).put(TELEMETRY_TRACING_ENABLED_SETTING.getKey(), false).build(); triggerUpdateConsumer(initial, update); verify(apmAgentSettings).setAgentSetting("recording", "true"); - verify(apmAgentSettings).setAgentSetting("instrument", "true"); verify(apmTelemetryProvider.getTracer()).setEnabled(true); } } @@ -76,7 +74,6 @@ public void testEnableTracingUsingLegacySetting() { apmAgentSettings.initAgentSystemProperties(settings); verify(apmAgentSettings).setAgentSetting("recording", "true"); - verify(apmAgentSettings).setAgentSetting("instrument", "true"); } public void testEnableMetrics() { @@ -90,7 +87,6 @@ public void testEnableMetrics() { apmAgentSettings.initAgentSystemProperties(update); verify(apmAgentSettings).setAgentSetting("recording", "true"); - verify(apmAgentSettings).setAgentSetting("instrument", Boolean.toString(tracingEnabled)); clearInvocations(apmAgentSettings); Settings initial = Settings.builder().put(update).put(TELEMETRY_METRICS_ENABLED_SETTING.getKey(), false).build(); @@ -114,13 +110,11 @@ public void testDisableTracing() { apmAgentSettings.initAgentSystemProperties(update); verify(apmAgentSettings).setAgentSetting("recording", Boolean.toString(metricsEnabled)); - verify(apmAgentSettings).setAgentSetting("instrument", "false"); clearInvocations(apmAgentSettings); Settings initial = Settings.builder().put(update).put(TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true).build(); triggerUpdateConsumer(initial, update); verify(apmAgentSettings).setAgentSetting("recording", Boolean.toString(metricsEnabled)); - verify(apmAgentSettings).setAgentSetting("instrument", "false"); verify(apmTelemetryProvider.getTracer()).setEnabled(false); } } @@ -130,7 +124,6 @@ public void testDisableTracingUsingLegacySetting() { apmAgentSettings.initAgentSystemProperties(settings); verify(apmAgentSettings).setAgentSetting("recording", "false"); - verify(apmAgentSettings).setAgentSetting("instrument", "false"); } public void testDisableMetrics() { @@ -144,7 +137,6 @@ public void testDisableMetrics() { apmAgentSettings.initAgentSystemProperties(update); verify(apmAgentSettings).setAgentSetting("recording", Boolean.toString(tracingEnabled)); - verify(apmAgentSettings).setAgentSetting("instrument", Boolean.toString(tracingEnabled)); clearInvocations(apmAgentSettings); Settings initial = Settings.builder().put(update).put(TELEMETRY_METRICS_ENABLED_SETTING.getKey(), true).build(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 694e015b602f8..88e529ec5569b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.PassThroughObjectMapper; import java.io.IOException; import java.io.UncheckedIOException; @@ -152,8 +153,9 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) // Avoid failing because index.routing_path is missing - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .putList(INDEX_ROUTING_PATH.getKey(), List.of("path")) .build(); tmpIndexMetadata.settings(finalResolvedSettings); @@ -164,6 +166,13 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { extractPath(routingPaths, fieldMapper); } + for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) { + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + if (passThroughObjectMapper.containsDimensions()) { + routingPaths.add(passThroughObjectMapper.fullPath() + ".*"); + } + } + } for (var template : mapperService.getAllDynamicTemplates()) { if (template.pathMatch().isEmpty()) { continue; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index b69ea170eb476..4cebba155518b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -62,6 +62,7 @@ import org.elasticsearch.datastreams.rest.RestMigrateToDataStreamAction; import org.elasticsearch.datastreams.rest.RestModifyDataStreamsAction; import org.elasticsearch.datastreams.rest.RestPromoteDataStreamAction; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.plugins.ActionPlugin; @@ -76,6 +77,7 @@ import java.util.Collection; import java.util.List; import java.util.Locale; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DATA_STREAM_LIFECYCLE_ORIGIN; @@ -237,7 +239,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { indexScopedSettings.addSettingsUpdateConsumer(LOOK_AHEAD_TIME, value -> { TimeValue timeSeriesPollInterval = updateTimeSeriesRangeService.get().pollInterval; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index db0e3e5cd6258..c65854903f7a9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -638,6 +638,33 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size()); } + public void testGenerateRoutingPathFromPassThroughObject() throws Exception { + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + String mapping = """ + { + "_doc": { + "properties": { + "labels": { + "type": "passthrough", + "time_series_dimension": true + }, + "metrics": { + "type": "passthrough" + }, + "another_field": { + "type": "keyword" + } + } + } + } + """; + Settings result = generateTsdbSettings(mapping, now); + assertThat(result.size(), equalTo(3)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); + } + private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { Metadata metadata = Metadata.EMPTY_METADATA; String dataStreamName = "logs-app1"; diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 6a84f08a2c193..81ede2b045e61 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -133,8 +133,8 @@ created the data stream: --- fetch the tsid: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -147,13 +147,13 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} --- "aggregate the tsid": - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.12.99" + reason: _tsid hahing introduced in 8.13 - do: search: @@ -168,9 +168,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -191,3 +191,358 @@ index without timestamp with pipeline: pipeline: my_pipeline body: - '{"@timestamp": "wrong_format", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + +--- +dynamic templates: + - skip: + version: " - 8.12.99" + features: "default_shards" + reason: "Support for dynamic fields was added in 8.13" + - do: + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.another.dim": "D" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.another.dim": "D" }' + + - do: + search: + index: k9s + body: + size: 0 + + - match: { hits.total.value: 4 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + dim: A + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur4gYpiKF9nXxRUMHnJBiqqP58deEIun8H8" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.dim: C + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur4gYpiKF9nXxRUMHnJBiqqP58deEIun8H8" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + +--- +dynamic templates - conflicting aliases: + - skip: + version: " - 8.12.99" + features: "default_shards" + reason: "Support for dynamic fields was added in 8.13" + - do: + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + resource_attributes: + type: passthrough + dynamic: true + time_series_dimension: true + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "resource_attributes.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "resource_attributes.dim": "C" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "resource_attributes.dim": "D" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "resource_attributes.dim": "D" }' + + - do: + search: + index: k9s + body: + size: 0 + + - match: { hits.total.value: 4 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + dim: "C" + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + attributes.dim: A + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + +--- +dynamic templates with nesting: + - skip: + version: " - 8.12.99" + features: "default_shards" + reason: "Support for dynamic fields was added in 8.13" + - do: + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + resource: + type: object + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }' + + - do: + search: + index: k9s + body: + size: 0 + + - match: { hits.total.value: 4 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + dim: A + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.dim: C + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + more.dim: E + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + +--- +dynamic templates - subobject in passthrough object error: + - skip: + version: " - 8.12.99" + reason: "Support for dynamic fields was added in 8.13" + - do: + catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/ + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + mode: time_series + + mappings: + properties: + attributes: + type: passthrough + properties: + subcategory: + type: object + properties: + dim: + type: keyword + + - do: + catch: /Mapping definition for \[attributes\] has unsupported parameters:\ \[subobjects \:\ true\]/ + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + subobjects: true diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml new file mode 100644 index 0000000000000..b9621977ff3aa --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml @@ -0,0 +1,110 @@ +--- +teardown: + - do: + indices.delete_data_stream: + name: logs-foobar + ignore: 404 + + - do: + indices.delete: + index: .fs-logs-foobar-* + ignore: 404 + + - do: + indices.delete_index_template: + name: generic_logs_template + ignore: 404 + + - do: + ingest.delete_pipeline: + id: "failing_pipeline" + ignore: 404 + +--- +"Redirect ingest failure in data stream to failure store": + - skip: + version: " - 8.12.99" + reason: "data stream failure stores only redirect ingest failures in 8.13+" + features: [allowed_warnings, contains] + + - do: + ingest.put_pipeline: + id: "failing_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "fail" : { + "message" : "error_message" + } + } + ] + } + - match: { acknowledged: true } + + - do: + allowed_warnings: + - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" + indices.put_index_template: + name: generic_logs_template + body: + index_patterns: logs-* + data_stream: + failure_store: true + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + index: + default_pipeline: "failing_pipeline" + + - do: + index: + index: logs-foobar + refresh: true + body: + '@timestamp': '2020-12-12' + foo: bar + + - do: + indices.get_data_stream: + name: logs-foobar + - match: { data_streams.0.name: logs-foobar } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store: true } + - length: { data_streams.0.failure_indices: 1 } + - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + + - do: + search: + index: logs-foobar + body: { query: { match_all: {} } } + - length: { hits.hits: 0 } + + - do: + search: + index: .fs-logs-foobar-* + - length: { hits.hits: 1 } + - match: { hits.hits.0._index: "/\\.fs-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" } + - exists: hits.hits.0._source.@timestamp + - not_exists: hits.hits.0._source.foo + - not_exists: hits.hits.0._source.document.id + - match: { hits.hits.0._source.document.index: 'logs-foobar' } + - match: { hits.hits.0._source.document.source.@timestamp: '2020-12-12' } + - match: { hits.hits.0._source.document.source.foo: 'bar' } + - match: { hits.hits.0._source.error.type: 'fail_processor_exception' } + - match: { hits.hits.0._source.error.message: 'error_message' } + - contains: { hits.hits.0._source.error.stack_trace: 'org.elasticsearch.ingest.common.FailProcessorException: error_message' } + + - do: + indices.delete_data_stream: + name: logs-foobar + - is_true: acknowledged + + - do: + indices.delete: + index: .fs-logs-foobar-* + - is_true: acknowledged diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml index 7aed1cbe0a636..df9f0840f575f 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_require_data_stream.yml @@ -60,10 +60,8 @@ --- "Testing require_data_stream in bulk requests": - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/104774" - #version: " - 8.12.99" - #reason: "require_data_stream was introduced in 8.13.0" + version: " - 8.12.99" + reason: "require_data_stream was introduced in 8.13.0" features: allowed_warnings - do: @@ -109,7 +107,7 @@ - do: allowed_warnings: - - "index template [other-template] has index patterns [ds-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [other-template] will take precedence during new index creation" + - "index template [other-template] has index patterns [other-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [other-template] will take precedence during new index creation" indices.put_index_template: name: other-template body: diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index dff65f1c7a1bc..9bf3c17e1ee63 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.ingest.DropProcessor; import org.elasticsearch.ingest.PipelineProcessor; import org.elasticsearch.ingest.Processor; @@ -28,6 +29,7 @@ import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Map.entry; @@ -90,7 +92,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new GrokProcessorGetAction.RestAction()); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 53c8db638923f..2e0a84cfde23b 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.Processor; @@ -52,6 +53,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; @@ -154,7 +156,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestGeoIpDownloaderStatsAction()); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index b7f5035122dfe..ebaa2f356733e 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; @@ -30,6 +31,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin, SearchPlugin { @@ -61,7 +63,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList( new RestSearchTemplateAction(namedWriteableRegistry), diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index f9deddd5f4e85..068821793e44c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.painless.action.PainlessContextAction; import org.elasticsearch.painless.action.PainlessExecuteAction; import org.elasticsearch.painless.spi.PainlessExtension; @@ -43,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; /** @@ -167,7 +169,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { List handlers = new ArrayList<>(); handlers.add(new PainlessExecuteAction.RestAction()); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index efdf3c09bbe92..d6eb55dfb23e4 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -21,7 +21,8 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MapperTestCase; +import org.elasticsearch.index.mapper.NumberFieldMapperTests; +import org.elasticsearch.index.mapper.NumberTypeOutOfRangeSpec; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -35,6 +36,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Function; @@ -45,7 +47,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notANumber; -public class ScaledFloatFieldMapperTests extends MapperTestCase { +public class ScaledFloatFieldMapperTests extends NumberFieldMapperTests { @Override protected Collection getPlugins() { @@ -199,7 +201,7 @@ public void testStore() throws Exception { assertEquals(1230, storedField.numericValue().longValue()); } - public void testCoerce() throws Exception { + public void testCoerce() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); ParsedDocument doc = mapper.parse( new SourceToParse( @@ -452,6 +454,41 @@ protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + @Override + protected List outOfRangeSpecs() { + // No outOfRangeSpecs are specified because ScaledFloatFieldMapper doesn't extend NumberFieldMapper and doesn't use a + // NumberFieldMapper.NumberType that is present in OutOfRangeSpecs + return Collections.emptyList(); + } + + @Override + public void testIgnoreMalformedWithObject() {} // TODO: either implement this, remove it, or update ScaledFloatFieldMapper's behaviour + + @Override + public void testAllowMultipleValuesField() {} // TODO: either implement this, remove it, or update ScaledFloatFieldMapper's behaviour + + @Override + public void testScriptableTypes() {} // TODO: either implement this, remove it, or update ScaledFloatFieldMapper's behaviour + + @Override + public void testDimension() {} // TODO: either implement this, remove it, or update ScaledFloatFieldMapper's behaviour + + @Override + protected Number missingValue() { + return 0.123; + } + + @Override + protected Number randomNumber() { + /* + * The source parser and doc values round trip will both reduce + * the precision to 32 bits if the value is more precise. + * randomDoubleBetween will smear the values out across a wide + * range of valid values. + */ + return randomBoolean() ? randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true) : randomFloat(); + } + public void testEncodeDecodeExactScalingFactor() { double v = randomValue(); assertThat(encodeDecode(1 / v, v), equalTo(1 / v)); diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java index 02776eb277020..dc0f3ea8bb8c6 100644 --- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java +++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java @@ -1404,6 +1404,7 @@ public void testParentChildQueriesViaScrollApi() throws Exception { for (QueryBuilder query : queries) { assertScrollResponsesAndHitCount( + client(), TimeValue.timeValueSeconds(60), prepareSearch("test").setScroll(TimeValue.timeValueSeconds(30)).setSize(1).addStoredField("_id").setQuery(query), 10, diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index 381fe2cd7a77e..f7214bfba1eba 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -28,6 +29,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class RankEvalPlugin extends Plugin implements ActionPlugin { @@ -48,7 +50,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Collections.singletonList(new RestRankEvalAction()); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 2a45b8e9d8be4..232a22239d901 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -61,7 +61,7 @@ public void testDCGAt() { SearchHit[] hits = new SearchHit[6]; for (int i = 0; i < 6; i++) { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); - hits[i] = new SearchHit(i, Integer.toString(i)); + hits[i] = SearchHit.unpooled(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -111,7 +111,7 @@ public void testDCGAtSixMissingRatings() { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } } - hits[i] = new SearchHit(i, Integer.toString(i)); + hits[i] = SearchHit.unpooled(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -168,7 +168,7 @@ public void testDCGAtFourMoreRatings() { // only create four hits SearchHit[] hits = new SearchHit[4]; for (int i = 0; i < 4; i++) { - hits[i] = new SearchHit(i, Integer.toString(i)); + hits[i] = SearchHit.unpooled(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java index 2f9d2a3a117ed..9b1319ae35055 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -104,7 +104,7 @@ private SearchHit[] createSearchHits(List rated, Integer[] releva if (relevanceRatings[i] != null) { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } - hits[i] = new SearchHit(i, Integer.toString(i)); + hits[i] = SearchHit.unpooled(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java index 1aa5df55f5296..1a42ba9606938 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java @@ -190,7 +190,7 @@ public void testXContentParsingIsNotLenient() throws IOException { private static SearchHit[] createSearchHits(int from, int to, String index) { SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { - hits[i] = new SearchHit(i, i + ""); + hits[i] = SearchHit.unpooled(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java index 2b199182619ce..10f22bc5a0b3f 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java @@ -101,7 +101,7 @@ public void testIgnoreUnlabeled() { rated.add(createRatedDoc("test", "1", RELEVANT_RATING)); // add an unlabeled search hit SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3); - searchHits[2] = new SearchHit(2, "2"); + searchHits[2] = SearchHit.unpooled(2, "2"); searchHits[2].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated); @@ -120,7 +120,7 @@ public void testIgnoreUnlabeled() { public void testNoRatedDocs() throws Exception { SearchHit[] hits = new SearchHit[5]; for (int i = 0; i < 5; i++) { - hits[i] = new SearchHit(i, i + ""); + hits[i] = SearchHit.unpooled(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList()); @@ -248,7 +248,7 @@ private static PrecisionAtK mutate(PrecisionAtK original) { private static SearchHit[] toSearchHits(List rated, String index) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i + ""); + hits[i] = SearchHit.unpooled(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index d4d58c3c0ae71..6631d3f80f1ad 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -226,7 +226,7 @@ public void testToXContent() throws IOException { } private static RatedSearchHit searchHit(String index, int docId, Integer rating) { - SearchHit hit = new SearchHit(docId, docId + ""); + SearchHit hit = SearchHit.unpooled(docId, docId + ""); hit.shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); hit.score(1.0f); return new RatedSearchHit(hit, rating != null ? OptionalInt.of(rating) : OptionalInt.empty()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index d38ae1f66fb1d..c5a09d67d94d0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.rankeval; -import org.apache.lucene.util.Constants; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; @@ -110,8 +109,6 @@ public static RatedRequest createTestItem(boolean forceRequest) { } public void testXContentRoundtrip() throws IOException { - assumeFalse("https://github.com/elastic/elasticsearch/issues/104570", Constants.WINDOWS); - RatedRequest testItem = createTestItem(randomBoolean()); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); @@ -126,8 +123,6 @@ public void testXContentRoundtrip() throws IOException { } public void testXContentParsingIsNotLenient() throws IOException { - assumeFalse("https://github.com/elastic/elasticsearch/issues/104570", Constants.WINDOWS); - RatedRequest testItem = createTestItem(randomBoolean()); XContentType xContentType = randomFrom(XContentType.values()); BytesReference originalBytes = toShuffledXContent(testItem, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()); @@ -266,8 +261,6 @@ public void testAggsNotAllowed() { } public void testSuggestionsNotAllowed() { - assumeFalse("https://github.com/elastic/elasticsearch/issues/104570", Constants.WINDOWS); - List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); SearchSourceBuilder query = new SearchSourceBuilder(); query.suggest(new SuggestBuilder().addSuggestion("id", SuggestBuilders.completionSuggestion("fieldname"))); @@ -306,8 +299,6 @@ public void testProfileNotAllowed() { * matter for parsing xContent */ public void testParseFromXContent() throws IOException { - assumeFalse("https://github.com/elastic/elasticsearch/issues/104570", Constants.WINDOWS); - String querySpecString = """ { "id": "my_qa_query", diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index d6cfb21049969..e124cc0354453 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -26,7 +26,7 @@ public class RatedSearchHitTests extends ESTestCase { public static RatedSearchHit randomRatedSearchHit() { OptionalInt rating = randomBoolean() ? OptionalInt.empty() : OptionalInt.of(randomIntBetween(0, 5)); - SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAlphaOfLength(10)); + SearchHit searchHit = SearchHit.unpooled(randomIntBetween(0, 10), randomAlphaOfLength(10)); RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating); return ratedSearchHit; } @@ -36,7 +36,7 @@ private static RatedSearchHit mutateTestItem(RatedSearchHit original) { SearchHit hit = original.getSearchHit(); switch (randomIntBetween(0, 1)) { case 0 -> rating = rating.isPresent() ? OptionalInt.of(rating.getAsInt() + 1) : OptionalInt.of(randomInt(5)); - case 1 -> hit = new SearchHit(hit.docId(), hit.getId() + randomAlphaOfLength(10)); + case 1 -> hit = SearchHit.unpooled(hit.docId(), hit.getId() + randomAlphaOfLength(10)); default -> throw new IllegalStateException("The test should only allow two parameters mutated"); } return new RatedSearchHit(hit, rating); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java index c5cbb84d66d2d..d7acd4a253c2c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java @@ -103,7 +103,7 @@ public void testNoRatedDocs() throws Exception { int k = 5; SearchHit[] hits = new SearchHit[k]; for (int i = 0; i < k; i++) { - hits[i] = new SearchHit(i, i + ""); + hits[i] = SearchHit.unpooled(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } @@ -217,7 +217,7 @@ private static RecallAtK mutate(RecallAtK original) { private static SearchHit[] toSearchHits(List rated, String index) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i + ""); + hits[i] = SearchHit.unpooled(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java index 95ee2a1ae2d6f..982d1afcf6dd3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.rankeval; -import org.apache.lucene.util.Constants; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; @@ -43,8 +42,6 @@ public final class TransportRankEvalActionTests extends ESTestCase { * Test that request parameters like indicesOptions or searchType from ranking evaluation request are transfered to msearch request */ public void testTransferRequestParameters() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/104570", Constants.WINDOWS); - String indexName = "test_index"; List specifications = new ArrayList<>(); specifications.add( diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java index 42ff1fda6e74d..88f22478d3cff 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.ReindexAction; @@ -35,6 +36,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.singletonList; @@ -70,7 +72,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList( new RestReindexAction(namedWriteableRegistry), diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml index df9d9d903efd6..98996cc9c24be 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml @@ -161,7 +161,7 @@ from tsdb to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -228,7 +228,7 @@ from standard with tsdb id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -295,7 +295,7 @@ from standard with random _id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -344,7 +344,7 @@ from tsdb to tsdb modifying timestamp: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-05-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-05-28T18:50:23.142Z} @@ -393,7 +393,7 @@ from tsdb to tsdb modifying dimension: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: bubbles}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy5k1S1a8G1YLZL5ZxtVAwhiHw5Ds"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -489,7 +489,7 @@ from tsdb to tsdb created by template while modifying dimension: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: bubbles}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy5k1S1a8G1YLZL5ZxtVAwhiHw5Ds"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 9ad2c57b7f585..1d9a5fee31d39 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -31,6 +31,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; @@ -72,6 +74,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -246,7 +249,7 @@ public void testMetrics() throws Exception { assertHitCount(prepareSearch(index).setSize(0).setTrackTotalHits(true), nbDocs); assertAcked(clusterAdmin().prepareDeleteSnapshot(repository, snapshot).get()); - final Map aggregatedMetrics = new HashMap<>(); + final Map aggregatedMetrics = new HashMap<>(); // Compare collected stats and metrics for each node and they should be the same for (var nodeName : internalCluster().getNodeNames()) { final BlobStoreRepository blobStoreRepository; @@ -293,13 +296,12 @@ public void testMetrics() throws Exception { metric.getLong(), equalTo(statsCollectors.get(statsKey).counter.sum()) ); - - aggregatedMetrics.compute(operation.getKey(), (k, v) -> v == null ? metric.getLong() : v + metric.getLong()); + aggregatedMetrics.compute(statsKey, (k, v) -> v == null ? metric.getLong() : v + metric.getLong()); }); } // Metrics number should be consistent with server side request count as well. - assertThat(aggregatedMetrics, equalTo(getMockRequestCounts())); + assertThat(aggregatedMetrics, equalTo(getServerMetrics())); } public void testRequestStatsWithOperationPurposes() throws IOException { @@ -423,6 +425,18 @@ public void testEnforcedCooldownPeriod() throws IOException { assertThat(repository.threadPool().relativeTimeInNanos() - beforeFastDelete, lessThan(TEST_COOLDOWN_PERIOD.getNanos())); } + private Map getServerMetrics() { + for (HttpHandler h : handlers.values()) { + while (h instanceof DelegatingHttpHandler) { + if (h instanceof S3StatsCollectorHttpHandler s3StatsCollectorHttpHandler) { + return Maps.transformValues(s3StatsCollectorHttpHandler.getMetricsCount(), AtomicLong::get); + } + h = ((DelegatingHttpHandler) h).getDelegate(); + } + } + return Collections.emptyMap(); + } + /** * S3RepositoryPlugin that allows to disable chunked encoding and to set a low threshold between single upload and multipart upload. */ @@ -525,13 +539,21 @@ protected String requestUniqueId(final HttpExchange exchange) { @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") protected class S3StatsCollectorHttpHandler extends HttpStatsCollectorHandler { + private final Map metricsCount = ConcurrentCollections.newConcurrentMap(); + S3StatsCollectorHttpHandler(final HttpHandler delegate) { super(delegate); } @Override public void handle(HttpExchange exchange) throws IOException { - final String request = exchange.getRequestMethod() + " " + exchange.getRequestURI(); + final S3HttpHandler.RequestComponents requestComponents = S3HttpHandler.parseRequestComponents( + S3HttpHandler.getRawRequestString(exchange) + ); + if (false == requestComponents.request().startsWith("HEAD ")) { + assertThat(requestComponents.customQueryParameters(), hasKey(S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE)); + } + final String request = requestComponents.request(); if (shouldFailCompleteMultipartUploadRequest.get() && Regex.simpleMatch("POST /*/*?uploadId=*", request)) { try (exchange) { drainInputStream(exchange.getRequestBody()); @@ -546,22 +568,53 @@ public void handle(HttpExchange exchange) throws IOException { } @Override - public void maybeTrack(final String request, Headers requestHeaders) { + public void maybeTrack(final String rawRequest, Headers requestHeaders) { + final S3HttpHandler.RequestComponents requestComponents = S3HttpHandler.parseRequestComponents(rawRequest); + final String request = requestComponents.request(); + final OperationPurpose purpose; + // TODO: Remove the condition once ES-7810 is resolved + if (false == request.startsWith("HEAD ")) { + purpose = OperationPurpose.parse( + requestComponents.customQueryParameters().get(S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE).get(0) + ); + } else { + purpose = null; + } if (Regex.simpleMatch("GET /*/?prefix=*", request)) { trackRequest("ListObjects"); + metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.LIST_OBJECTS, purpose), k -> new AtomicLong()) + .incrementAndGet(); } else if (Regex.simpleMatch("GET /*/*", request)) { trackRequest("GetObject"); + metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.GET_OBJECT, purpose), k -> new AtomicLong()) + .incrementAndGet(); } else if (isMultiPartUpload(request)) { trackRequest("PutMultipartObject"); + metricsCount.computeIfAbsent( + new S3BlobStore.StatsKey(S3BlobStore.Operation.PUT_MULTIPART_OBJECT, purpose), + k -> new AtomicLong() + ).incrementAndGet(); } else if (Regex.simpleMatch("PUT /*/*", request)) { trackRequest("PutObject"); + metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.PUT_OBJECT, purpose), k -> new AtomicLong()) + .incrementAndGet(); } else if (Regex.simpleMatch("POST /*/?delete", request)) { trackRequest("DeleteObjects"); + metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.DELETE_OBJECTS, purpose), k -> new AtomicLong()) + .incrementAndGet(); } else if (Regex.simpleMatch("DELETE /*/*?uploadId=*", request)) { trackRequest("AbortMultipartObject"); + metricsCount.computeIfAbsent( + new S3BlobStore.StatsKey(S3BlobStore.Operation.ABORT_MULTIPART_OBJECT, purpose), + k -> new AtomicLong() + ).incrementAndGet(); } } + Map getMetricsCount() { + return metricsCount; + } + private boolean isMultiPartUpload(String request) { return Regex.simpleMatch("POST /*/*?uploads", request) || Regex.simpleMatch("POST /*/*?*uploadId=*", request) diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index dadd15ed640c0..b70fd8e87eeef 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -90,6 +90,7 @@ class S3BlobContainer extends AbstractBlobContainer { @Override public boolean blobExists(OperationPurpose purpose, String blobName) { + // TODO: Exists request needs to be include for metrics as well, see ES-7810 try (AmazonS3Reference clientReference = blobStore.clientReference()) { return SocketAccess.doPrivileged(() -> clientReference.client().doesObjectExist(blobStore.bucket(), buildKey(blobName))); } catch (final Exception e) { @@ -207,7 +208,7 @@ protected void onCompletion() throws IOException { uploadId.get(), parts ); - complRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.PUT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(complRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose); SocketAccess.doPrivilegedVoid(() -> clientReference.client().completeMultipartUpload(complRequest)); } } @@ -240,7 +241,7 @@ private UploadPartRequest createPartUploadRequest( uploadRequest.setUploadId(uploadId); uploadRequest.setPartNumber(number); uploadRequest.setInputStream(stream); - uploadRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.PUT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(uploadRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose); uploadRequest.setPartSize(size); uploadRequest.setLastPart(lastPart); return uploadRequest; @@ -248,7 +249,7 @@ private UploadPartRequest createPartUploadRequest( private void abortMultiPartUpload(OperationPurpose purpose, String uploadId, String blobName) { final AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(blobStore.bucket(), blobName, uploadId); - abortRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.ABORT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(abortRequest, blobStore, Operation.ABORT_MULTIPART_OBJECT, purpose); try (AmazonS3Reference clientReference = blobStore.clientReference()) { SocketAccess.doPrivilegedVoid(() -> clientReference.client().abortMultipartUpload(abortRequest)); } @@ -258,7 +259,7 @@ private InitiateMultipartUploadRequest initiateMultiPartUpload(OperationPurpose final InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(blobStore.bucket(), blobName); initRequest.setStorageClass(blobStore.getStorageClass()); initRequest.setCannedACL(blobStore.getCannedACL()); - initRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.PUT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(initRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose); if (blobStore.serverSideEncryption()) { final ObjectMetadata md = new ObjectMetadata(); md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); @@ -289,13 +290,13 @@ public DeleteResult delete(OperationPurpose purpose) throws IOException { final ObjectListing list; if (prevListing != null) { final var listNextBatchOfObjectsRequest = new ListNextBatchOfObjectsRequest(prevListing); - listNextBatchOfObjectsRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.LIST_OBJECTS, purpose)); + S3BlobStore.configureRequestForMetrics(listNextBatchOfObjectsRequest, blobStore, Operation.LIST_OBJECTS, purpose); list = SocketAccess.doPrivileged(() -> clientReference.client().listNextBatchOfObjects(listNextBatchOfObjectsRequest)); } else { final ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(blobStore.bucket()); listObjectsRequest.setPrefix(keyPath); - listObjectsRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.LIST_OBJECTS, purpose)); + S3BlobStore.configureRequestForMetrics(listObjectsRequest, blobStore, Operation.LIST_OBJECTS, purpose); list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest)); } final Iterator blobNameIterator = Iterators.map(list.getObjectSummaries().iterator(), summary -> { @@ -378,7 +379,7 @@ private List executeListing( ObjectListing list; if (prevListing != null) { final var listNextBatchOfObjectsRequest = new ListNextBatchOfObjectsRequest(prevListing); - listNextBatchOfObjectsRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.LIST_OBJECTS, purpose)); + S3BlobStore.configureRequestForMetrics(listNextBatchOfObjectsRequest, blobStore, Operation.LIST_OBJECTS, purpose); list = SocketAccess.doPrivileged(() -> clientReference.client().listNextBatchOfObjects(listNextBatchOfObjectsRequest)); } else { list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest)); @@ -394,10 +395,11 @@ private List executeListing( } private ListObjectsRequest listObjectsRequest(OperationPurpose purpose, String pathPrefix) { - return new ListObjectsRequest().withBucketName(blobStore.bucket()) + final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(blobStore.bucket()) .withPrefix(pathPrefix) - .withDelimiter("/") - .withRequestMetricCollector(blobStore.getMetricCollector(Operation.LIST_OBJECTS, purpose)); + .withDelimiter("/"); + S3BlobStore.configureRequestForMetrics(listObjectsRequest, blobStore, Operation.LIST_OBJECTS, purpose); + return listObjectsRequest; } // exposed for tests @@ -432,7 +434,7 @@ void executeSingleUpload( final PutObjectRequest putRequest = new PutObjectRequest(s3BlobStore.bucket(), blobName, input, md); putRequest.setStorageClass(s3BlobStore.getStorageClass()); putRequest.setCannedAcl(s3BlobStore.getCannedACL()); - putRequest.setRequestMetricCollector(s3BlobStore.getMetricCollector(Operation.PUT_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(putRequest, blobStore, Operation.PUT_OBJECT, purpose); try (AmazonS3Reference clientReference = s3BlobStore.clientReference()) { SocketAccess.doPrivilegedVoid(() -> { clientReference.client().putObject(putRequest); }); @@ -510,7 +512,7 @@ void executeMultipartUpload( uploadId.get(), parts ); - complRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.PUT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(complRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose); SocketAccess.doPrivilegedVoid(() -> clientReference.client().completeMultipartUpload(complRequest)); success = true; @@ -708,7 +710,7 @@ private void logUploads(String description, List uploads) { private List listMultipartUploads() { final var listRequest = new ListMultipartUploadsRequest(bucket); listRequest.setPrefix(blobKey); - listRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.LIST_OBJECTS, purpose)); + S3BlobStore.configureRequestForMetrics(listRequest, blobStore, Operation.LIST_OBJECTS, purpose); try { return SocketAccess.doPrivileged(() -> client.listMultipartUploads(listRequest)).getMultipartUploads(); } catch (AmazonS3Exception e) { @@ -721,7 +723,7 @@ private List listMultipartUploads() { private String initiateMultipartUpload() { final var initiateRequest = new InitiateMultipartUploadRequest(bucket, blobKey); - initiateRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.PUT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(initiateRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose); return SocketAccess.doPrivileged(() -> client.initiateMultipartUpload(initiateRequest)).getUploadId(); } @@ -734,7 +736,7 @@ private PartETag uploadPart(BytesReference updated, String uploadId) throws IOEx uploadPartRequest.setLastPart(true); uploadPartRequest.setInputStream(updated.streamInput()); uploadPartRequest.setPartSize(updated.length()); - uploadPartRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.PUT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(uploadPartRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose); return SocketAccess.doPrivileged(() -> client.uploadPart(uploadPartRequest)).getPartETag(); } @@ -828,7 +830,7 @@ private void safeAbortMultipartUpload(String uploadId) { private void abortMultipartUploadIfExists(String uploadId) { try { final var request = new AbortMultipartUploadRequest(bucket, blobKey, uploadId); - request.setRequestMetricCollector(blobStore.getMetricCollector(Operation.ABORT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(request, blobStore, Operation.ABORT_MULTIPART_OBJECT, purpose); SocketAccess.doPrivilegedVoid(() -> client.abortMultipartUpload(request)); } catch (AmazonS3Exception e) { if (e.getStatusCode() != 404) { @@ -840,7 +842,7 @@ private void abortMultipartUploadIfExists(String uploadId) { private void completeMultipartUpload(String uploadId, PartETag partETag) { final var completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucket, blobKey, uploadId, List.of(partETag)); - completeMultipartUploadRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.PUT_MULTIPART_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(completeMultipartUploadRequest, blobStore, Operation.PUT_MULTIPART_OBJECT, purpose); SocketAccess.doPrivilegedVoid(() -> client.completeMultipartUpload(completeMultipartUploadRequest)); } } @@ -875,7 +877,7 @@ public void compareAndExchangeRegister( public void getRegister(OperationPurpose purpose, String key, ActionListener listener) { ActionListener.completeWith(listener, () -> { final var getObjectRequest = new GetObjectRequest(blobStore.bucket(), buildKey(key)); - getObjectRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.GET_OBJECT, purpose)); + S3BlobStore.configureRequestForMetrics(getObjectRequest, blobStore, Operation.GET_OBJECT, purpose); try ( var clientReference = blobStore.clientReference(); var s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest)); diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index c045e05a6f8e0..68def0598ef60 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories.s3; import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.Request; import com.amazonaws.Response; import com.amazonaws.metrics.RequestMetricCollector; @@ -55,6 +56,8 @@ class S3BlobStore implements BlobStore { + public static final String CUSTOM_QUERY_PARAMETER_PURPOSE = "x-purpose"; + /** * Maximum number of deletes in a {@link DeleteObjectsRequest}. * @see S3 Documentation. @@ -85,10 +88,6 @@ class S3BlobStore implements BlobStore { private final StatsCollectors statsCollectors = new StatsCollectors(); - private static final TimeValue RETRY_STATS_WINDOW = TimeValue.timeValueMinutes(5); - - private volatile S3RequestRetryStats s3RequestRetryStats; - S3BlobStore( S3Service service, String bucket, @@ -112,23 +111,10 @@ class S3BlobStore implements BlobStore { this.threadPool = threadPool; this.snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT); this.repositoriesMetrics = repositoriesMetrics; - s3RequestRetryStats = new S3RequestRetryStats(getMaxRetries()); - threadPool.scheduleWithFixedDelay(() -> { - var priorRetryStats = s3RequestRetryStats; - s3RequestRetryStats = new S3RequestRetryStats(getMaxRetries()); - priorRetryStats.emitMetrics(); - }, RETRY_STATS_WINDOW, threadPool.generic()); } RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) { - var collector = statsCollectors.getMetricCollector(operation, purpose); - return new RequestMetricCollector() { - @Override - public void collectMetrics(Request request, Response response) { - s3RequestRetryStats.addRequest(request); - collector.collectMetrics(request, response); - } - }; + return statsCollectors.getMetricCollector(operation, purpose); } public Executor getSnapshotExecutor() { @@ -358,9 +344,11 @@ private void deletePartition( } private static DeleteObjectsRequest bulkDelete(OperationPurpose purpose, S3BlobStore blobStore, List blobs) { - return new DeleteObjectsRequest(blobStore.bucket()).withKeys(blobs.toArray(Strings.EMPTY_ARRAY)) - .withQuiet(true) - .withRequestMetricCollector(blobStore.getMetricCollector(Operation.DELETE_OBJECTS, purpose)); + final DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(blobStore.bucket()).withKeys( + blobs.toArray(Strings.EMPTY_ARRAY) + ).withQuiet(true); + configureRequestForMetrics(deleteObjectsRequest, blobStore, Operation.DELETE_OBJECTS, purpose); + return deleteObjectsRequest; } @Override @@ -473,4 +461,14 @@ IgnoreNoResponseMetricsCollector buildMetricCollector(Operation operation, Opera return new IgnoreNoResponseMetricsCollector(operation, purpose); } } + + static void configureRequestForMetrics( + AmazonWebServiceRequest request, + S3BlobStore blobStore, + Operation operation, + OperationPurpose purpose + ) { + request.setRequestMetricCollector(blobStore.getMetricCollector(operation, purpose)); + request.putCustomQueryParameter(CUSTOM_QUERY_PARAMETER_PURPOSE, purpose.getKey()); + } } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java deleted file mode 100644 index b7c37c6d95fde..0000000000000 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.repositories.s3; - -import com.amazonaws.Request; -import com.amazonaws.util.AWSRequestMetrics; -import com.amazonaws.util.TimingInfo; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.logging.ESLogMessage; -import org.elasticsearch.common.util.Maps; - -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicLongArray; - -/** - * This class emit aws s3 metrics as logs until we have a proper apm integration - */ -public class S3RequestRetryStats { - public static final String MESSAGE_FIELD = "message"; - - private static final Logger logger = LogManager.getLogger(S3RequestRetryStats.class); - - private final AtomicLong requests = new AtomicLong(); - private final AtomicLong exceptions = new AtomicLong(); - private final AtomicLong throttles = new AtomicLong(); - private final AtomicLongArray exceptionsHistogram; - private final AtomicLongArray throttlesHistogram; - - public S3RequestRetryStats(int maxRetries) { - this.exceptionsHistogram = new AtomicLongArray(maxRetries + 1); - this.throttlesHistogram = new AtomicLongArray(maxRetries + 1); - } - - public void addRequest(Request request) { - if (request == null) { - return; - } - var info = request.getAWSRequestMetrics().getTimingInfo(); - long requests = getCounter(info, AWSRequestMetrics.Field.RequestCount); - long exceptions = getCounter(info, AWSRequestMetrics.Field.Exception); - long throttles = getCounter(info, AWSRequestMetrics.Field.ThrottleException); - - this.requests.addAndGet(requests); - this.exceptions.addAndGet(exceptions); - this.throttles.addAndGet(throttles); - if (exceptions >= 0 && exceptions < this.exceptionsHistogram.length()) { - this.exceptionsHistogram.incrementAndGet((int) exceptions); - } - if (throttles >= 0 && throttles < this.throttlesHistogram.length()) { - this.throttlesHistogram.incrementAndGet((int) throttles); - } - } - - private static long getCounter(TimingInfo info, AWSRequestMetrics.Field field) { - var counter = info.getCounter(field.name()); - return counter != null ? counter.longValue() : 0L; - } - - public void emitMetrics() { - if (logger.isDebugEnabled()) { - var metrics = Maps.newMapWithExpectedSize(4); - metrics.put(MESSAGE_FIELD, "S3 Request Retry Stats"); - metrics.put("elasticsearch.metrics.s3.requests", requests.get()); - metrics.put("elasticsearch.metrics.s3.exceptions", exceptions.get()); - metrics.put("elasticsearch.metrics.s3.throttles", throttles.get()); - for (int i = 0; i < exceptionsHistogram.length(); i++) { - long exceptions = exceptionsHistogram.get(i); - if (exceptions != 0) { - metrics.put("elasticsearch.metrics.s3.exceptions_histogram_" + i, exceptions); - } - } - for (int i = 0; i < throttlesHistogram.length(); i++) { - long throttles = throttlesHistogram.get(i); - if (throttles != 0) { - metrics.put("elasticsearch.metrics.s3.throttles_histogram_" + i, throttles); - } - } - logger.debug(new ESLogMessage().withFields(metrics)); - } - } -} diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java index 0ebf6c54b49aa..c457b9d51e8b9 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java @@ -29,6 +29,7 @@ import java.util.List; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.repositories.s3.S3BlobStore.configureRequestForMetrics; /** * Wrapper around an S3 object that will retry the {@link GetObjectRequest} if the download fails part-way through, resuming from where @@ -86,7 +87,7 @@ private void openStreamWithRetry() throws IOException { while (true) { try (AmazonS3Reference clientReference = blobStore.clientReference()) { final GetObjectRequest getObjectRequest = new GetObjectRequest(blobStore.bucket(), blobKey); - getObjectRequest.setRequestMetricCollector(blobStore.getMetricCollector(Operation.GET_OBJECT, purpose)); + configureRequestForMetrics(getObjectRequest, blobStore, Operation.GET_OBJECT, purpose); if (currentOffset > 0 || start > 0 || end < Long.MAX_VALUE - 1) { assert start + currentOffset <= end : "requesting beyond end, start = " + start + " offset=" + currentOffset + " end=" + end; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 04a836997e0f7..0ddd29171b3bd 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -7,6 +7,8 @@ */ package org.elasticsearch.repositories.s3; +import fixture.s3.S3HttpHandler; + import com.amazonaws.DnsResolver; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -230,7 +232,10 @@ public void testWriteBlobWithRetries() throws Exception { final byte[] bytes = randomBlobContent(); httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_max_retries"), exchange -> { - if ("PUT".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getQuery() == null) { + final S3HttpHandler.RequestComponents requestComponents = S3HttpHandler.parseRequestComponents( + S3HttpHandler.getRawRequestString(exchange) + ); + if ("PUT".equals(requestComponents.method()) && requestComponents.query().isEmpty()) { if (countDown.countDown()) { final BytesReference body = Streams.readFully(exchange.getRequestBody()); if (Objects.deepEquals(bytes, BytesReference.toBytes(body))) { @@ -319,9 +324,12 @@ public void testWriteLargeBlob() throws Exception { final CountDown countDownComplete = new CountDown(nbErrors); httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_large_blob"), exchange -> { + final S3HttpHandler.RequestComponents requestComponents = S3HttpHandler.parseRequestComponents( + S3HttpHandler.getRawRequestString(exchange) + ); final long contentLength = Long.parseLong(exchange.getRequestHeaders().getFirst("Content-Length")); - if ("POST".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getQuery().equals("uploads")) { + if ("POST".equals(requestComponents.method()) && requestComponents.query().equals("uploads")) { // initiate multipart upload request if (countDownInitiate.countDown()) { byte[] response = (""" @@ -337,9 +345,9 @@ public void testWriteLargeBlob() throws Exception { exchange.close(); return; } - } else if ("PUT".equals(exchange.getRequestMethod()) - && exchange.getRequestURI().getQuery().contains("uploadId=TEST") - && exchange.getRequestURI().getQuery().contains("partNumber=")) { + } else if ("PUT".equals(requestComponents.method()) + && requestComponents.query().contains("uploadId=TEST") + && requestComponents.query().contains("partNumber=")) { // upload part request MD5DigestCalculatingInputStream md5 = new MD5DigestCalculatingInputStream(exchange.getRequestBody()); BytesReference bytes = Streams.readFully(md5); @@ -353,7 +361,7 @@ public void testWriteLargeBlob() throws Exception { return; } - } else if ("POST".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getQuery().equals("uploadId=TEST")) { + } else if ("POST".equals(requestComponents.method()) && requestComponents.query().equals("uploadId=TEST")) { // complete multipart upload request if (countDownComplete.countDown()) { Streams.readFully(exchange.getRequestBody()); @@ -418,9 +426,12 @@ public void testWriteLargeBlobStreaming() throws Exception { final CountDown countDownComplete = new CountDown(nbErrors); httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_large_blob_streaming"), exchange -> { + final S3HttpHandler.RequestComponents requestComponents = S3HttpHandler.parseRequestComponents( + S3HttpHandler.getRawRequestString(exchange) + ); final long contentLength = Long.parseLong(exchange.getRequestHeaders().getFirst("Content-Length")); - if ("POST".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getQuery().equals("uploads")) { + if ("POST".equals(requestComponents.method()) && requestComponents.query().equals("uploads")) { // initiate multipart upload request if (countDownInitiate.countDown()) { byte[] response = (""" @@ -436,9 +447,9 @@ public void testWriteLargeBlobStreaming() throws Exception { exchange.close(); return; } - } else if ("PUT".equals(exchange.getRequestMethod()) - && exchange.getRequestURI().getQuery().contains("uploadId=TEST") - && exchange.getRequestURI().getQuery().contains("partNumber=")) { + } else if ("PUT".equals(requestComponents.method()) + && requestComponents.query().contains("uploadId=TEST") + && requestComponents.query().contains("partNumber=")) { // upload part request MD5DigestCalculatingInputStream md5 = new MD5DigestCalculatingInputStream(exchange.getRequestBody()); BytesReference bytes = Streams.readFully(md5); @@ -451,7 +462,7 @@ public void testWriteLargeBlobStreaming() throws Exception { return; } - } else if ("POST".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getQuery().equals("uploadId=TEST")) { + } else if ("POST".equals(requestComponents.method()) && requestComponents.query().equals("uploadId=TEST")) { // complete multipart upload request if (countDownComplete.countDown()) { Streams.readFully(exchange.getRequestBody()); diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java index ad7b821c986c1..7b234dbbe9177 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/MainRestPlugin.java @@ -18,12 +18,14 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class MainRestPlugin extends Plugin implements ActionPlugin { @@ -39,7 +41,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestMainAction()); } diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java index 8cd68abdcce42..b62af9791bcec 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedEncodingIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.BaseRestHandler; @@ -44,6 +45,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -107,7 +109,8 @@ public Collection getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new BaseRestHandler() { @Override diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java index ad205c6f28783..33fdb00e7abb2 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java @@ -165,7 +165,7 @@ public void sendMessage(BytesReference reference, ActionListener listener) // We need to both guard against double resolving the listener and not resolving it in case of event loop shutdown so we need to // use #notifyOnce here until https://github.com/netty/netty/issues/8007 is resolved. var wrapped = ActionListener.notifyOnce(listener); - channel.writeAndFlush(Netty4Utils.toByteBuf(reference), addPromise(wrapped, channel)); + channel.writeAndFlush(reference, addPromise(wrapped, channel)); if (channel.eventLoop().isShutdown()) { wrapped.onFailure(new TransportException("Cannot send message, event loop is shutting down.")); } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java index 2dae8bc9258fe..b9986dbf00d87 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java @@ -72,9 +72,13 @@ public static void setAvailableProcessors(final int availableProcessors) { * pages of the BytesReference. Don't free the bytes of reference before the ByteBuf goes out of scope. */ public static ByteBuf toByteBuf(final BytesReference reference) { - if (reference.length() == 0) { - return Unpooled.EMPTY_BUFFER; + if (reference.hasArray()) { + return Unpooled.wrappedBuffer(reference.array(), reference.arrayOffset(), reference.length()); } + return compositeReferenceToByteBuf(reference); + } + + private static ByteBuf compositeReferenceToByteBuf(BytesReference reference) { final BytesRefIterator iterator = reference.iterator(); // usually we have one, two, or three components from the header, the message, and a buffer final List buffers = new ArrayList<>(3); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java index 3246c52e08bd0..ced2d7d65fa16 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandler.java @@ -9,6 +9,7 @@ package org.elasticsearch.transport.netty4; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelFuture; @@ -16,12 +17,17 @@ import io.netty.channel.ChannelPromise; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; +import io.netty.util.concurrent.PromiseCombiner; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.transport.Transports; +import java.io.IOException; import java.nio.channels.ClosedChannelException; -import java.util.ArrayDeque; +import java.util.LinkedList; import java.util.Queue; /** @@ -32,7 +38,7 @@ public final class Netty4WriteThrottlingHandler extends ChannelDuplexHandler { public static final int MAX_BYTES_PER_WRITE = 1 << 18; - private final Queue queuedWrites = new ArrayDeque<>(); + private final Queue queuedWrites = new LinkedList<>(); private final ThreadContext threadContext; private WriteOperation currentWrite; @@ -42,17 +48,36 @@ public Netty4WriteThrottlingHandler(ThreadContext threadContext) { } @Override - public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) { - assert msg instanceof ByteBuf; + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws IOException { + if (msg instanceof BytesReference reference) { + if (reference.hasArray()) { + writeSingleByteBuf(ctx, Unpooled.wrappedBuffer(reference.array(), reference.arrayOffset(), reference.length()), promise); + } else { + BytesRefIterator iter = reference.iterator(); + final PromiseCombiner combiner = new PromiseCombiner(ctx.executor()); + BytesRef next; + while ((next = iter.next()) != null) { + final ChannelPromise chunkPromise = ctx.newPromise(); + combiner.add((Future) chunkPromise); + writeSingleByteBuf(ctx, Unpooled.wrappedBuffer(next.bytes, next.offset, next.length), chunkPromise); + } + combiner.finish(promise); + } + } else { + assert msg instanceof ByteBuf; + writeSingleByteBuf(ctx, (ByteBuf) msg, promise); + } + } + + private void writeSingleByteBuf(ChannelHandlerContext ctx, ByteBuf buf, ChannelPromise promise) { assert Transports.assertDefaultThreadContext(threadContext); assert Transports.assertTransportThread(); - final ByteBuf buf = (ByteBuf) msg; if (ctx.channel().isWritable() && currentWrite == null && queuedWrites.isEmpty()) { // nothing is queued for writing and the channel is writable, just pass the write down the pipeline directly if (buf.readableBytes() > MAX_BYTES_PER_WRITE) { writeInSlices(ctx, promise, buf); } else { - ctx.write(msg, promise); + ctx.write(buf, promise); } } else { queueWrite(buf, promise); diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java index 59828649c58fb..8ac39d925f566 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java @@ -14,6 +14,9 @@ import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.ESTestCase; @@ -28,6 +31,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.oneOf; public class Netty4WriteThrottlingHandlerTests extends ESTestCase { @@ -56,42 +60,76 @@ public void testThrottlesLargeMessage() throws ExecutionException, InterruptedEx assertThat(writeableBytes, lessThan(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); final int fullSizeChunks = randomIntBetween(2, 10); final int extraChunkSize = randomIntBetween(0, 10); - final ByteBuf message = Unpooled.wrappedBuffer( - randomByteArrayOfLength(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE * fullSizeChunks + extraChunkSize) + final byte[] messageBytes = randomByteArrayOfLength( + Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE * fullSizeChunks + extraChunkSize ); + final Object message = wrapAsNettyOrEsBuffer(messageBytes); final ChannelPromise promise = embeddedChannel.newPromise(); transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); assertThat(seen, hasSize(1)); - assertEquals(message.slice(0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE), seen.get(0)); + assertSliceEquals(seen.get(0), message, 0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE); assertFalse(promise.isDone()); transportGroup.getLowLevelGroup().submit(embeddedChannel::flush).get(); assertTrue(promise.isDone()); assertThat(seen, hasSize(fullSizeChunks + (extraChunkSize == 0 ? 0 : 1))); assertTrue(capturingHandler.didWriteAfterThrottled); if (extraChunkSize != 0) { - assertEquals( - message.slice(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE * fullSizeChunks, extraChunkSize), - seen.get(seen.size() - 1) + assertSliceEquals( + seen.get(seen.size() - 1), + message, + Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE * fullSizeChunks, + extraChunkSize ); } } - public void testPassesSmallMessageDirectly() throws ExecutionException, InterruptedException { + public void testThrottleLargeCompositeMessage() throws ExecutionException, InterruptedException { final List seen = new CopyOnWriteArrayList<>(); final CapturingHandler capturingHandler = new CapturingHandler(seen); final EmbeddedChannel embeddedChannel = new EmbeddedChannel( capturingHandler, new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY)) ); + // we assume that the channel outbound buffer is smaller than Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE final int writeableBytes = Math.toIntExact(embeddedChannel.bytesBeforeUnwritable()); assertThat(writeableBytes, lessThan(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); - final ByteBuf message = Unpooled.wrappedBuffer( - randomByteArrayOfLength(randomIntBetween(0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)) + final int fullSizeChunks = randomIntBetween(2, 10); + final int extraChunkSize = randomIntBetween(0, 10); + final byte[] messageBytes = randomByteArrayOfLength( + Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE * fullSizeChunks + extraChunkSize + ); + int splitOffset = randomIntBetween(0, messageBytes.length); + final BytesReference message = CompositeBytesReference.of( + new BytesArray(messageBytes, 0, splitOffset), + new BytesArray(messageBytes, splitOffset, messageBytes.length - splitOffset) + ); + final ChannelPromise promise = embeddedChannel.newPromise(); + transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); + assertThat(seen, hasSize(oneOf(1, 2))); + assertSliceEquals(seen.get(0), message, 0, seen.get(0).readableBytes()); + assertFalse(promise.isDone()); + transportGroup.getLowLevelGroup().submit(embeddedChannel::flush).get(); + assertTrue(promise.isDone()); + assertThat(seen, hasSize(oneOf(fullSizeChunks, fullSizeChunks + 1))); + assertTrue(capturingHandler.didWriteAfterThrottled); + assertBufferEquals(Unpooled.compositeBuffer().addComponents(true, seen), message); + } + + public void testPassesSmallMessageDirectly() throws ExecutionException, InterruptedException { + final List seen = new CopyOnWriteArrayList<>(); + final CapturingHandler capturingHandler = new CapturingHandler(seen); + final EmbeddedChannel embeddedChannel = new EmbeddedChannel( + capturingHandler, + new Netty4WriteThrottlingHandler(new ThreadContext(Settings.EMPTY)) ); + final int writeableBytes = Math.toIntExact(embeddedChannel.bytesBeforeUnwritable()); + assertThat(writeableBytes, lessThan(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); + final byte[] messageBytes = randomByteArrayOfLength(randomIntBetween(0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); + final Object message = wrapAsNettyOrEsBuffer(messageBytes); final ChannelPromise promise = embeddedChannel.newPromise(); transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); assertThat(seen, hasSize(1)); // first message should be passed through straight away - assertSame(message, seen.get(0)); + assertBufferEquals(seen.get(0), message); assertFalse(promise.isDone()); transportGroup.getLowLevelGroup().submit(embeddedChannel::flush).get(); assertTrue(promise.isDone()); @@ -107,13 +145,14 @@ public void testThrottlesOnUnwritable() throws ExecutionException, InterruptedEx ); final int writeableBytes = Math.toIntExact(embeddedChannel.bytesBeforeUnwritable()); assertThat(writeableBytes, lessThan(Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)); - final ByteBuf message = Unpooled.wrappedBuffer(randomByteArrayOfLength(writeableBytes + randomIntBetween(0, 10))); + final byte[] messageBytes = randomByteArrayOfLength(writeableBytes + randomIntBetween(0, 10)); + final Object message = wrapAsNettyOrEsBuffer(messageBytes); final ChannelPromise promise = embeddedChannel.newPromise(); transportGroup.getLowLevelGroup().submit(() -> embeddedChannel.write(message, promise)).get(); assertThat(seen, hasSize(1)); // first message should be passed through straight away - assertSame(message, seen.get(0)); + assertBufferEquals(seen.get(0), message); assertFalse(promise.isDone()); - final ByteBuf messageToQueue = Unpooled.wrappedBuffer( + final Object messageToQueue = wrapAsNettyOrEsBuffer( randomByteArrayOfLength(randomIntBetween(0, Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE)) ); final ChannelPromise promiseForQueued = embeddedChannel.newPromise(); @@ -126,6 +165,31 @@ public void testThrottlesOnUnwritable() throws ExecutionException, InterruptedEx assertTrue(promiseForQueued.isDone()); } + private static void assertBufferEquals(ByteBuf expected, Object message) { + if (message instanceof ByteBuf buf) { + assertSame(expected, buf); + } else { + assertEquals(expected, Netty4Utils.toByteBuf(asInstanceOf(BytesReference.class, message))); + } + } + + private static void assertSliceEquals(ByteBuf expected, Object message, int index, int length) { + assertEquals( + (message instanceof ByteBuf buf ? buf : Netty4Utils.toByteBuf(asInstanceOf(BytesReference.class, message))).slice( + index, + length + ), + expected + ); + } + + private static Object wrapAsNettyOrEsBuffer(byte[] messageBytes) { + if (randomBoolean()) { + return Unpooled.wrappedBuffer(messageBytes); + } + return new BytesArray(messageBytes); + } + private static class CapturingHandler extends ChannelOutboundHandlerAdapter { private final List seen; diff --git a/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java b/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java index e142ba80147e0..a820973c19ca3 100644 --- a/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java +++ b/plugins/examples/rest-handler/src/main/java/org/elasticsearch/example/resthandler/ExampleRestHandlerPlugin.java @@ -15,12 +15,14 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.singletonList; @@ -35,8 +37,8 @@ public List getRestHandlers(final Settings settings, final IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter, final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster) { - + final Supplier nodesInCluster, + final Predicate clusterSupportsFeature) { return singletonList(new ExampleCatAction()); } } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java index 273196f392064..82485130f05ce 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.rest.RestTestLegacyFeatures; import org.elasticsearch.xcontent.XContentBuilder; @@ -215,6 +216,31 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio } private static final List TSDB_DIMS = List.of("6a841a21", "947e4ced", "a4c385a1", "b47a2f4e", "df3145b3"); + + private static final List EXPECTED_TSDB_TSIDS_NODES_0 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); + + private static final List EXPECTED_TSDB_TSIDS_NODES_1 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf1qe4IAVEfi3wL8wnx3pd_rA41HpA", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); + + private static final List EXPECTED_TSDB_TSIDS_NODES_2 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf1fK0gnGg01X2P0BuJX0wb-7i2pqA", + "JFFUy14C9UcX3MnFnsFrpf1qe4IAVEfi3wL8wnx3pd_rA41HpA", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); + private static final List EXPECTED_TSDB_TSIDS_NODES_3 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf0ayKYYMlhssuNhG-tPuituN3POiA", + "JFFUy14C9UcX3MnFnsFrpf1fK0gnGg01X2P0BuJX0wb-7i2pqA", + "JFFUy14C9UcX3MnFnsFrpf1qe4IAVEfi3wL8wnx3pd_rA41HpA", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); private static final long[] TSDB_TIMES; static { String[] times = new String[] { @@ -230,6 +256,7 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio } public void testTsdb() throws IOException { + final Version oldClusterVersion = Version.fromString(getOldClusterVersion()); assumeTrue("indexing time series indices changed in 8.2.0", oldClusterHasFeature(RestTestLegacyFeatures.TSDB_NEW_INDEX_FORMAT)); StringBuilder bulk = new StringBuilder(); @@ -238,23 +265,54 @@ public void testTsdb() throws IOException { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[0], TSDB_TIMES[1], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[0], TSDB_TIMES[1], -0.1); bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(215.95, 0.005), closeTo(-215.95, 0.005)); - return; + assertTsdbAgg(oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_0, closeTo(215.95, 0.005), closeTo(-215.95, 0.005)); } else if (isFirstMixedCluster()) { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[1], TSDB_TIMES[2], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1); tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1); bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(217.45, 0.005), closeTo(-217.45, 0.005), closeTo(2391.95, 0.005)); - + if (oldClusterVersion.onOrAfter(Version.V_8_13_0)) { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_1, + closeTo(217.45, 0.005), + closeTo(2391.95, 0.005), + closeTo(-217.45, 0.005) + ); + } else { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_1, + closeTo(217.45, 0.005), + closeTo(-217.45, 0.005), + closeTo(2391.95, 0.005) + ); + } } else if (isMixedCluster()) { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[2], TSDB_TIMES[3], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[2], TSDB_TIMES[3], -0.1); tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1); tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10); bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(218.95, 0.005), closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), closeTo(21895, 0.5)); - return; + if (oldClusterVersion.onOrAfter(Version.V_8_13_0)) { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_2, + closeTo(218.95, 0.5), + closeTo(21895.0, 0.005), + closeTo(2408.45, 0.005), + closeTo(-218.95, 0.005) + ); + } else { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_2, + closeTo(218.95, 0.005), + closeTo(-218.95, 0.005), + closeTo(2408.45, 0.005), + closeTo(21895, 0.5) + ); + } } else { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[3], TSDB_TIMES[4], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[3], TSDB_TIMES[4], -0.1); @@ -262,13 +320,27 @@ public void testTsdb() throws IOException { tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10); tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5); bulk("tsdb", bulk.toString()); - assertTsdbAgg( - closeTo(220.45, 0.005), - closeTo(-220.45, 0.005), - closeTo(2424.95, 0.005), - closeTo(22045, 0.5), - closeTo(-11022.5, 0.5) - ); + if (oldClusterVersion.onOrAfter(Version.V_8_13_0)) { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_3, + closeTo(220.45, 0.005), + closeTo(-11022.5, 0.5), + closeTo(22045, 0.5), + closeTo(2424.95, 0.005), + closeTo(-220.45, 0.005) + ); + } else { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_3, + closeTo(220.45, 0.005), + closeTo(-220.45, 0.005), + closeTo(2424.95, 0.005), + closeTo(22045, 0.5), + closeTo(-11022.5, 0.5) + ); + } } } @@ -310,13 +382,15 @@ private void tsdbBulk(StringBuilder bulk, String dim, long timeStart, long timeE } } - private void assertTsdbAgg(Matcher... expected) throws IOException { + private void assertTsdbAgg(final Version oldClusterVersion, final List expectedTsids, final Matcher... expected) + throws IOException { + boolean onOrAfterTsidHashingVersion = oldClusterVersion.onOrAfter(Version.V_8_13_0); Request request = new Request("POST", "/tsdb/_search"); request.addParameter("size", "0"); XContentBuilder body = JsonXContent.contentBuilder().startObject(); body.startObject("aggs").startObject("tsids"); { - body.startObject("terms").field("field", "_tsid").endObject(); + body.startObject("terms").field("field", TimeSeriesIdFieldMapper.NAME).endObject(); body.startObject("aggs").startObject("avg"); { body.startObject("avg").field("field", "value").endObject(); @@ -327,7 +401,8 @@ private void assertTsdbAgg(Matcher... expected) throws IOException { request.setJsonEntity(Strings.toString(body.endObject())); ListMatcher tsidsExpected = matchesList(); for (int d = 0; d < expected.length; d++) { - Object key = Map.of("dim", TSDB_DIMS.get(d)); + // NOTE: from Version 8.12.0 on we use tsid hashing for the _tsid field + Object key = onOrAfterTsidHashingVersion ? expectedTsids.get(d) : Map.of("dim", IndexingIT.TSDB_DIMS.get(d)); tsidsExpected = tsidsExpected.item(matchesMap().extraOk().entry("key", key).entry("avg", Map.of("value", expected[d]))); } assertMap( diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java index 081135d6b1e17..f130856265a32 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor.Type; import org.elasticsearch.plugins.Plugin; @@ -43,6 +44,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -160,7 +162,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new AddDocRestHandler()); } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java index 5c01d0fd430b4..4820202fc4d96 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/TestResponseHeaderPlugin.java @@ -15,12 +15,14 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.singletonList; @@ -35,7 +37,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return singletonList(new TestResponseHeaderRestAction()); } diff --git a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java index 971d1bad3e976..b86aefb12a956 100644 --- a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java +++ b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -36,6 +37,7 @@ import java.io.UncheckedIOException; import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; @@ -134,7 +136,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new CreateNetNewSystemIndexHandler(), new IndexDocHandler()); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json index bc8f12a933b1e..562190f6f5cad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json @@ -31,6 +31,14 @@ "type": "int", "default": 100, "description": "specifies a max number of results to get (default: 100)" + }, + "index_name": { + "type": "string", + "description": "connector index name(s) to fetch connector documents for" + }, + "connector_name": { + "type": "string", + "description": "connector name(s) to fetch connector documents for" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_api_key_id.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_api_key_id.json new file mode 100644 index 0000000000000..5b58a7b5b59a5 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_api_key_id.json @@ -0,0 +1,38 @@ +{ + "connector.update_api_key_id": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/connector-apis.html", + "description": "Updates the API key id and/or API key secret id fields in the connector document." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_connector/{connector_id}/_api_key_id", + "methods": [ + "PUT" + ], + "parts": { + "connector_id": { + "type": "string", + "description": "The unique identifier of the connector to be updated." + } + } + } + ] + }, + "body": { + "description": "An object containing the connector's API key id and/or Connector Secret document id for that API key.", + "required": true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_index_name.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_index_name.json new file mode 100644 index 0000000000000..92efe70a736b9 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_index_name.json @@ -0,0 +1,38 @@ +{ + "connector.update_index_name": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/connector-apis.html", + "description": "Updates the index name of the connector." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_connector/{connector_id}/_index_name", + "methods": [ + "PUT" + ], + "parts": { + "connector_id": { + "type": "string", + "description": "The unique identifier of the connector to be updated." + } + } + } + ] + }, + "body": { + "description": "An object containing the connector's index name.", + "required": true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_status.json new file mode 100644 index 0000000000000..159dae2e969ab --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_status.json @@ -0,0 +1,38 @@ +{ + "connector.update_status": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/connector-apis.html", + "description": "Updates the status of the connector." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_connector/{connector_id}/_status", + "methods": [ + "PUT" + ], + "parts": { + "connector_id": { + "type": "string", + "description": "The unique identifier of the connector to be updated." + } + } + } + ] + }, + "body": { + "description": "An object containing the connector's status.", + "required": true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/security.query_user.json b/rest-api-spec/src/main/resources/rest-api-spec/api/security.query_user.json new file mode 100644 index 0000000000000..6d76126ba81c4 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/security.query_user.json @@ -0,0 +1,33 @@ +{ + "security.query_user": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-query-user.html", + "description": "Retrieves information for Users using a subset of query DSL" + }, + "stability": "stable", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_security/_query/user", + "methods": [ + "GET", + "POST" + ] + } + ] + }, + "body": { + "description": "From, size, query, sort and search_after", + "required": false + } + } +} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 130f3690bb298..83f8aab34e02a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -1,14 +1,8 @@ ---- -setup: - - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - --- "basic tsdb delete": - skip: - version: " - 8.8.0" - reason: fixed in 8.8.1 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -49,15 +43,15 @@ setup: location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: crxuhC8WO3aVdhvtAAABiHD35_g - - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } + id: crxuhAep5Npwt_etAAABiHD35_g + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: deleted } - match: { _version: 2 } @@ -74,6 +68,6 @@ setup: location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 3 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/150_knn_search_missing_params.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/150_knn_search_missing_params.yml new file mode 100644 index 0000000000000..23c6b62842e9f --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/150_knn_search_missing_params.yml @@ -0,0 +1,139 @@ +setup: + - skip: + version: ' - 8.12.99' + reason: '[k] and [num_candidates] were made optional for kNN search in 8.13.0' + - do: + indices.create: + index: knn_search_test_index + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + + - do: + index: + index: knn_search_test_index + id: "1" + body: + vector: [1.0, -10.5, 1.3, 0.593, 41] + + - do: + index: + index: knn_search_test_index + id: "2" + body: + vector: [-0.5, 100.0, -13, 14.8, -156.0] + + - do: + index: + index: knn_search_test_index + id: "3" + body: + vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + - do: + indices.refresh: {} + +--- +"kNN with missing k param using default size": + - do: + search: + rest_total_hits_as_int: true + index: knn_search_test_index + body: + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + num_candidates: 10 + + - match: {hits.total: 3} + +--- +"kNN with missing k param using provided size": + - do: + search: + rest_total_hits_as_int: true + index: knn_search_test_index + body: + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + num_candidates: 10 + size: 2 + + - match: {hits.total: 2} + +--- +"kNN search with missing num_candidates param": + + - do: + search: + rest_total_hits_as_int: true + index: knn_search_test_index + body: + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + + - match: {hits.total: 2} + +--- +"kNN search with missing both k and num_candidates param - default size": + - do: + search: + rest_total_hits_as_int: true + index: knn_search_test_index + body: + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + + - match: {hits.total: 3} + + +--- +"kNN search with missing both k and num_candidates param - provided size": + + - do: + search: + rest_total_hits_as_int: true + index: knn_search_test_index + body: + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + size: 2 + + - match: {hits.total: 2} + +--- +"kNN search with missing k, and num_candidates < size": + + - do: + catch: bad_request + search: + index: knn_search_test_index + body: + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + num_candidates: 2 + size: 10 + +--- +"kNN search with missing k, default size, and invalid num_candidates": + + - do: + catch: bad_request + search: + index: knn_search_test_index + body: + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + num_candidates: 2 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml new file mode 100644 index 0000000000000..5194c95151eda --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml @@ -0,0 +1,172 @@ +setup: + - skip: + version: ' - 8.12.99' + reason: '[k] and [num_candidates] were made optional for kNN query in 8.13.0' + - do: + indices.create: + index: knn_query_test_index + body: + mappings: + properties: + vector: + type: dense_vector + dims: 3 + index: true + similarity: l2_norm + category: + type: keyword + nested: + type: nested + properties: + paragraph_id: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + + - do: + index: + index: knn_query_test_index + id: "1" + body: + vector: [1.0, 1.0, 0.0] + category: A + nested: + - paragraph_id: 0 + vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] + - paragraph_id: 1 + vector: [ 240.0, 300, -3, 1, -20 ] + + - do: + index: + index: knn_query_test_index + id: "2" + body: + vector: [1.0, 0.5, 1.0] + category: A + nested: + - paragraph_id: 2 + vector: [ 0, 100.0, 0, 14.8, -156.0 ] + + - do: + index: + index: knn_query_test_index + id: "3" + body: + vector: [-1, -1, -1] + category: B + nested: + - paragraph_id: 0 + vector: [ 100, 200.0, 300, 14.8, -156.0 ] + + - do: + indices.refresh: {} + +--- +"kNN query with missing num_candidates param - default size": + + - do: + search: + rest_total_hits_as_int: true + index: knn_query_test_index + body: + query: + knn: + field: vector + query_vector: [0, 0, 0] + + - match: { hits.total: 3 } + +--- +"kNN query with missing num_candidates param - size provided": + - do: + search: + rest_total_hits_as_int: true + index: knn_query_test_index + body: + query: + knn: + field: vector + query_vector: [1, 1, 1] + size: 1 + - match: { hits.total: 2 } # due to num_candidates defined as round(1.5 * size), so we only see 2 results + - length: { hits.hits: 1 } # one result is only returned though + +--- +"kNN query with num_candidates less than size": + + - do: + search: + rest_total_hits_as_int: true + index: knn_query_test_index + body: + query: + knn: + field: vector + query_vector: [-1, -1, -1] + num_candidates: 1 + size: 10 + + - match: { hits.total: 1 } + + +--- +"kNN query in a bool clause - missing num_candidates": + - do: + search: + rest_total_hits_as_int: true + index: knn_query_test_index + body: + query: + bool: + must: + - term: + category: A + - knn: + field: vector + query_vector: [ 1, 1, 0] + size: 1 + + - match: { hits.total: 2 } # due to num_candidates defined as round(1.5 * size), so we only see 2 results from cat:A + - length: { hits.hits: 1 } + +--- +"kNN search in a dis_max query - missing num_candidates": + - do: + search: + index: knn_query_test_index + body: + query: + dis_max: + queries: + - knn: + field: vector + query_vector: [1, 1, 0] + - match: + category: B + tie_breaker: 0.8 + size: 1 + + - match: { hits.total.value: 3 } # 2 knn result + 1 extra from match query + - length: { hits.hits: 1 } + +--- +"kNN search used in nested field - missing num_candidates": + - do: + search: + index: knn_query_test_index + body: + query: + nested: + path: nested + query: + knn: + field: nested.vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + inner_hits: { size: 1, "fields": [ "nested.paragraph_id" ], _source: false } + size: 1 + + - match: { hits.total.value: 2 } + - length: { hits.hits: 1 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml new file mode 100644 index 0000000000000..7da00a02d4285 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_flat.yml @@ -0,0 +1,274 @@ +setup: + - skip: + version: ' - 8.12.99' + reason: 'kNN flat index added in 8.13' + - do: + indices.create: + index: flat + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + index_options: + type: flat + another_vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + index_options: + type: flat + + - do: + index: + index: flat + id: "1" + body: + name: cow.jpg + vector: [230.0, 300.33, -34.8988, 15.555, -200.0] + another_vector: [130.0, 115.0, -1.02, 15.555, -100.0] + + - do: + index: + index: flat + id: "2" + body: + name: moose.jpg + vector: [-0.5, 100.0, -13, 14.8, -156.0] + another_vector: [-0.5, 50.0, -1, 1, 120] + + - do: + index: + index: flat + id: "3" + body: + name: rabbit.jpg + vector: [0.5, 111.3, -13.0, 14.8, -156.0] + another_vector: [-0.5, 11.0, 0, 12, 111.0] + + - do: + indices.refresh: {} + +--- +"kNN search only": + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search only": + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} +--- +"kNN search plus query": + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + query: + term: + name: cow.jpg + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search with query": + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + query: + term: + name: cow.jpg + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1.fields.name.0: "cow.jpg"} + + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2.fields.name.0: "moose.jpg"} +--- +"kNN search with filter": + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + filter: + term: + name: "rabbit.jpg" + + - match: {hits.total.value: 1} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: {hits.total.value: 0} + +--- +"KNN Vector similarity search only": + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} +--- +"Vector similarity with filter only": + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + filter: {"term": {"name": "moose.jpg"}} + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - do: + search: + index: flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 110 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + filter: {"term": {"name": "cow.jpg"}} + + - length: {hits.hits: 0} +--- +"Cosine similarity with indexed vector": + - skip: + features: "headers" + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "cosineSimilarity(params.query_vector, 'vector')" + params: + query_vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "3"} + - gte: {hits.hits.0._score: 0.999} + - lte: {hits.hits.0._score: 1.001} + + - match: {hits.hits.1._id: "2"} + - gte: {hits.hits.1._score: 0.998} + - lte: {hits.hits.1._score: 1.0} + + - match: {hits.hits.2._id: "1"} + - gte: {hits.hits.2._score: 0.78} + - lte: {hits.hits.2._score: 0.791} +--- +"Test bad parameters": + - do: + catch: bad_request + indices.create: + index: bad_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + index: true + index_options: + type: flat + m: 42 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml new file mode 100644 index 0000000000000..81d49dad21a70 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml @@ -0,0 +1,289 @@ +setup: + - skip: + version: ' - 8.12.99' + reason: 'kNN int8_flat index added in 8.13' + - do: + indices.create: + index: int8_flat + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + index_options: + type: int8_flat + another_vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + index_options: + type: int8_flat + + - do: + index: + index: int8_flat + id: "1" + body: + name: cow.jpg + vector: [230.0, 300.33, -34.8988, 15.555, -200.0] + another_vector: [130.0, 115.0, -1.02, 15.555, -100.0] + + - do: + index: + index: int8_flat + id: "2" + body: + name: moose.jpg + vector: [-0.5, 100.0, -13, 14.8, -156.0] + another_vector: [-0.5, 50.0, -1, 1, 120] + + - do: + index: + index: int8_flat + id: "3" + body: + name: rabbit.jpg + vector: [0.5, 111.3, -13.0, 14.8, -156.0] + another_vector: [-0.5, 11.0, 0, 12, 111.0] + + - do: + indices.refresh: {} + +--- +"kNN search only": + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search only": + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} +--- +"kNN search plus query": + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + query: + term: + name: cow.jpg + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search with query": + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + query: + term: + name: cow.jpg + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1.fields.name.0: "cow.jpg"} + + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2.fields.name.0: "moose.jpg"} +--- +"kNN search with filter": + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + filter: + term: + name: "rabbit.jpg" + + - match: {hits.total.value: 1} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: {hits.total.value: 0} + +--- +"KNN Vector similarity search only": + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 10.3 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} +--- +"Vector similarity with filter only": + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + filter: {"term": {"name": "moose.jpg"}} + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - do: + search: + index: int8_flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 110 + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + filter: {"term": {"name": "cow.jpg"}} + + - length: {hits.hits: 0} +--- +"Cosine similarity with indexed vector": + - skip: + features: "headers" + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "cosineSimilarity(params.query_vector, 'vector')" + params: + query_vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "3"} + - gte: {hits.hits.0._score: 0.999} + - lte: {hits.hits.0._score: 1.001} + + - match: {hits.hits.1._id: "2"} + - gte: {hits.hits.1._score: 0.998} + - lte: {hits.hits.1._score: 1.0} + + - match: {hits.hits.2._id: "1"} + - gte: {hits.hits.2._score: 0.78} + - lte: {hits.hits.2._score: 0.791} +--- +"Test bad parameters": + - do: + catch: bad_request + indices.create: + index: bad_int8_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + index: true + index_options: + type: int8_flat + m: 42 + + - do: + catch: bad_request + indices.create: + index: bad_int8_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + element_type: byte + index: true + index_options: + type: int8_flat diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 19d3defba1a4a..e546c60c5916a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -64,8 +64,8 @@ setup: --- composite aggregation on tsid: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -91,31 +91,31 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid.k8s\.pod\.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.0.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { aggregations.tsids.buckets.0.key.date: 1619635800000} - match: { aggregations.tsids.buckets.0.doc_count: 3 } - - match: { aggregations.tsids.buckets.1.key.tsid.k8s\.pod\.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.1.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.1.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { aggregations.tsids.buckets.1.key.date: 1619635860000} - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - match: { aggregations.tsids.buckets.2.key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.2.key.tsid.metricset: "pod" } - - match: { aggregations.tsids.buckets.2.key.date: 1619635800000} + + - match: { aggregations.tsids.buckets.2.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } + - match: { aggregations.tsids.buckets.2.key.date: 1619635800000 } - match: { aggregations.tsids.buckets.2.doc_count: 3 } - - match: { aggregations.tsids.buckets.3.key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.3.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.3.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.buckets.3.key.date: 1619635860000} - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.after_key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.after_key.date: 1619635860000} --- composite aggregation on tsid with after: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -139,17 +139,17 @@ composite aggregation on tsid with after: } ] after: { - tsid: { k8s.pod.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9", metricset: "pod" }, + tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", date: 1619635800000 } - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 1 } - - match: { aggregations.tsids.buckets.0.key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.0.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.buckets.0.key.date: 1619635860000} - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.after_key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.after_key.date: 1619635860000} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index deaba75e5476a..5813445326ef6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -1,16 +1,9 @@ ---- -setup: - - skip: - version: " - 8.9.99" - reason: "counter field support added in 8.10" - features: close_to - --- missing routing path field: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -78,25 +71,21 @@ missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.0.key.tag: null } + - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.15, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.1.key.tag: null } + - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.69, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.2.key.tag: "first" } + - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.3.key.tag: "second" } + - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.70, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} --- missing dimension on routing path field: @@ -133,8 +122,8 @@ missing dimension on routing path field: multi-value routing path field: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -209,12 +198,10 @@ multi-value routing path field: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 2} - - match: {aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: {aggregations.tsids.buckets.0.key.tag: "first" } + - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.1.key.tag: "second" } + - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 6.70, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 5b9ece81155e5..6ef03ba8ebcc4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.9.99" - reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: "- 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs4NdV58ePSPIAAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} - do: bulk: @@ -85,13 +85,59 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs4NdV58ePSPIAAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + + - do: + search: + index: test + body: + query: + match_all: {} + sort: ["@timestamp"] + + - match: {hits.total.value: 9} + + - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } + - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } + - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } + - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } + - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } + - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } + - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } + - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } --- index a new document on top of an old one: - skip: - version: " - 8.1.99" - reason: indexing on top of another document support added in 8.2 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -124,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4exTOUtxytuLkQAAABeRnR_mY} + - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -135,11 +181,11 @@ index a new document on top of an old one: max_tx: max: field: k8s.pod.network.tx - max_rx: + min_rx: min: field: k8s.pod.network.rx - match: {aggregations.max_tx.value: 1.11434595272E11} - - match: {aggregations.max_rx.value: 4.30605511E8} + - match: {aggregations.min_rx.value: 4.30605511E8} --- index a new document on top of an old one over bulk: @@ -169,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4exTOUtxytuLkQAAABeRnR_mY} + - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -193,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4exTOUtxytuLkQAAABeRnR_mY\\]\\[\\{.+\\}\\@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -212,8 +258,8 @@ create operation on top of old document fails: --- create operation on top of old document fails over bulk: - skip: - version: " - 8.1.99" - reason: id generation changed in 8.2 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: bulk: @@ -222,13 +268,13 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{k8s.pod.uid=df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset=pod}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: - skip: - version: " - 8.1.99" - reason: ids generation changed in 8.2 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -238,26 +284,26 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["cn4exTOUtxytuLkQAAABeRnR_mY", "cZZNs4NdV58ePSPIAAABeRnSA5M"] + values: ["cn4excfoxSs_KdA5AAABeRnR11Y", "cn4excfoxSs_KdA5AAABeRnR_mY"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exTOUtxytuLkQAAABeRnR_mY"} - - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} - - match: {hits.hits.1._id: "cZZNs4NdV58ePSPIAAABeRnSA5M"} - - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} + - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} + - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} + - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} --- get: - skip: - version: " - 8.1.99" - reason: ids generation changed in 8.2 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: get: index: test - id: cZZNs4NdV58ePSPIAAABeRnSA5M + id: cZZNs7B9sSWsyrL5AAABeRnSA5M - match: {_index: test} - - match: {_id: cZZNs4NdV58ePSPIAAABeRnSA5M} + - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -293,19 +339,19 @@ get with routing: catch: bad_request get: index: test - id: cZZNs4NdV58ePSPIAAABeRnSA5M + id: cZZNs-xII2fZweptAAABeRnSA5M routing: routing --- delete: - skip: - version: " - 8.1.99" - reason: ids generation changed in 8.2 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: delete: index: test - id: cZZNs4NdV58ePSPIAAABeRnSA5M + id: cn4excfoxSs_KdA5AAABeRnR_mY - match: {result: deleted} --- @@ -345,20 +391,20 @@ delete over _bulk: mget: index: test body: - ids: [ cn4exTOUtxytuLkQAAABeRnR_mY, cZZNs4NdV58ePSPIAAABeRnSA5M ] + ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "cn4exTOUtxytuLkQAAABeRnR_mY" } + - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "cZZNs4NdV58ePSPIAAABeRnSA5M" } + - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } - match: { docs.1.found: true } - do: bulk: index: test body: - - '{"delete": {"_id": "cn4exTOUtxytuLkQAAABeRnR_mY"}}' - - '{"delete": {"_id": "cZZNs4NdV58ePSPIAAABeRnSA5M"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} @@ -368,8 +414,8 @@ delete over _bulk: --- routing_path matches deep object: - skip: - version: " - 8.1.99" - reason: id generation changed in 8.2 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -408,13 +454,13 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGaxBa0saxogMAAABeRnRGTM} + - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} --- routing_path matches object: - skip: - version: " - 8.1.99" - reason: id generation changed in 8.2 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -449,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqUyQKH6n8noAAAABeRnRGTM} + - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 6fbaa2751345e..9d27507d0e32b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: snapshot.create_repository: @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [ { k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod } ] } + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 23f08170d4f69..5b779894a2cb1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.9.99" - reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.1.99,8.7.00 - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -171,8 +171,8 @@ fetch a tag: --- "fetch the tsid": - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -185,7 +185,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} --- aggregate a dimension: @@ -266,8 +266,8 @@ aggregate a tag: --- "aggregate the tsid": - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -282,9 +282,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -309,17 +309,26 @@ aggregate a tag: --- sort by tsid: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: index: test body: + fields: [ "_tsid" ] sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: [{ "k8s.pod.uid" : "947e4ced-1786-4e53-9e0c-5c447e959507", "metricset" : "pod"}, 1619635804467]} - - match: {hits.hits.1.sort: [{ "k8s.pod.uid" : "947e4ced-1786-4e53-9e0c-5c447e959507", "metricset" : "pod"}, 1619635824467]} - - match: {hits.hits.4.sort: [{ "k8s.pod.uid" : "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "metricset" : "pod"}, 1619635803142]} - - match: {hits.hits.7.sort: [{ "k8s.pod.uid" : "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "metricset" : "pod"}, 1619635863142]} + + - match: {hits.hits.0.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ", 1619635803142]} + - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"]} + + - match: {hits.hits.1.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ", 1619635823142]} + - match: {hits.hits.1.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"]} + + - match: {hits.hits.4.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", 1619635804467]} + - match: {hits.hits.4.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} + + - match: {hits.hits.7.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", 1619635864467]} + - match: {hits.hits.7.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index 128ba3fe3e464..829e26a90805a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -64,8 +64,8 @@ setup: --- search an alias: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.put_alias: @@ -85,16 +85,16 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- index into alias: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.put_alias: @@ -129,10 +129,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.2.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 969df4381bc0c..b30c03c4797da 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -1,14 +1,8 @@ ---- -setup: - - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - --- add dimensions with put_mapping: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -54,14 +48,14 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat } ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- add dimensions to no dims with dynamic_template over index: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -102,14 +96,14 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat } ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- add dimensions to no dims with dynamic_template over bulk: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -150,14 +144,14 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat } ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- add dimensions to some dims with dynamic_template over index: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -202,14 +196,14 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat, other_dim: cat } ] } + - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- add dimensions to some dims with dynamic_template over bulk: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -253,5 +247,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat, other_dim: cat } ] } + - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index f6c5ce2404a2a..6398fee265fef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -1,15 +1,9 @@ ---- -setup: - - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - --- keyword dimension: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: @@ -71,19 +65,19 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {uid: 947e4ced-1786-4e53-9e0c-5c447e959507}} + - match: {aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9}} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} + - match: {aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 3.3, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} --- flattened dimension: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: @@ -150,44 +144,28 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: "1516op6778" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.patch: "0" } + - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.09, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: "1516op6885" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.patch: "0" } + - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.35, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: "16w3xaca09" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.patch: "1" } + - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.64, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: "16w3xacq34" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.patch: "1" } + - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} --- flattened empty dimension: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: @@ -254,21 +232,21 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } + - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.22, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } + - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.69, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} --- flattened field missing routing path field: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -334,54 +312,36 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: "8" } + - match: { aggregations.tsids.buckets.0.key.: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: "8" } + - match: { aggregations.tsids.buckets.1.key: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.69, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: "1516op6778" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.09, error: 0.01 }} + - match: { aggregations.tsids.buckets.2.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } + - match: { aggregations.tsids.buckets.2.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: "1516op6885" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.40, error: 0.01 }} - - - match: { aggregations.tsids.buckets.4.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: "16w3xaca09" } - - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.4.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 6.59, error: 0.01 }} - - - match: { aggregations.tsids.buckets.5.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: "16w3xacq34" } - - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.5.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.75, error: 0.01 }} + - match: { aggregations.tsids.buckets.3.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } + - match: { aggregations.tsids.buckets.3.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} + + - match: { aggregations.tsids.buckets.4.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } + - match: { aggregations.tsids.buckets.4.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} + + - match: { aggregations.tsids.buckets.5.key: "MF1xFkQfNVXn4EbbAW53ge_xTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } + - match: { aggregations.tsids.buckets.5.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.60, error: 0.01 }} --- flattened field misspelled routing path field: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -452,28 +412,28 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: "1516op6778" } - - match: { aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.09, error: 0.01 }} + - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } + - match: { aggregations.tsids.buckets.0.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: "1516op6885" } - - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.40, error: 0.01 }} + - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } + - match: { aggregations.tsids.buckets.1.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: "16w3xaca09" } - - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.59, error: 0.01 }} + - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } + - match: { aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: "16w3xacq34" } - - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} + - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } + - match: { aggregations.tsids.buckets.3.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} --- long dimension: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -536,19 +496,19 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {id: 1, metricset: aa}} + - match: {aggregations.tsids.buckets.0.key: "KMaueSdBhc_WIhY4xoPE2EdDgKYd73outpXn7LJV-gQfvlrec7NyMho"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {id: 2, metricset: aa }} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} + - match: {aggregations.tsids.buckets.1.key: "KMaueSdBhc_WIhY4xoPE2Eetm41v73outmoSTcFmfQBYjOjMaOWM5zs"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 3.3, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} --- ip dimension: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -611,12 +571,12 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { ip: "10.10.1.1", metricset: aa}} + - match: {aggregations.tsids.buckets.0.key: "KLVrd7E8oZLyd-tfSm0H6jDYzb-v73outosB-eDb_nzTrIVsJFVQR2c"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { ip: "2001:db8:85a3::8a2e:370:7334", metricset: aa }} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} + - match: {aggregations.tsids.buckets.1.key: "KLVrd7E8oZLyd-tfSm0H6jDaaOFK73outkqpG8R65Gm4VUhpyuc11zw"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 3.3, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} --- runtime time series dimension: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 9720ead43fd98..486e5bf8dc607 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.9.99" - reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.1.99,8.7.00 - 8.12.99" + reason: _tsid hashing introduced in 8.13 features: "arbitrary_key" # Force allocating all shards to a single node so that we can shrink later. @@ -102,8 +102,8 @@ split: --- shrink: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.shrink: @@ -124,13 +124,13 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} --- clone: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.clone: @@ -148,4 +148,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 884f6dbcd677e..0766b732099c4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -48,10 +48,10 @@ import org.elasticsearch.tasks.TaskResult; import org.elasticsearch.tasks.TaskResultsService; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.tasks.MockTaskManager; import org.elasticsearch.test.tasks.MockTaskManagerListener; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ReceiveTimeoutTransportException; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentType; @@ -64,6 +64,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; @@ -71,6 +72,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -531,22 +533,32 @@ public void testTasksUnblocking() throws Exception { ); } - @TestLogging( - reason = "https://github.com/elastic/elasticsearch/issues/97923", - value = "org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction:TRACE" - ) public void testListTasksWaitForCompletion() throws Exception { - waitForCompletionTestCase( - randomBoolean(), - id -> clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).setWaitForCompletion(true).execute(), - response -> { - assertThat(response.getNodeFailures(), empty()); - assertThat(response.getTaskFailures(), empty()); - assertThat(response.getTasks(), hasSize(1)); - TaskInfo task = response.getTasks().get(0); - assertEquals(TEST_TASK_ACTION.name(), task.action()); + waitForCompletionTestCase(randomBoolean(), id -> { + var future = ensureStartedOnAllNodes( + "cluster:monitor/tasks/lists[n]", + () -> clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).setWaitForCompletion(true).execute() + ); + + // This ensures that a task has progressed to the point of listing all running tasks and subscribing to their updates + for (var threadPool : internalCluster().getInstances(ThreadPool.class)) { + var max = threadPool.info(ThreadPool.Names.MANAGEMENT).getMax(); + var executor = threadPool.executor(ThreadPool.Names.MANAGEMENT); + var waitForManagementToCompleteAllTasks = new CyclicBarrier(max + 1); + for (int i = 0; i < max; i++) { + executor.submit(() -> safeAwait(waitForManagementToCompleteAllTasks)); + } + safeAwait(waitForManagementToCompleteAllTasks); } - ); + + return future; + }, response -> { + assertThat(response.getNodeFailures(), empty()); + assertThat(response.getTaskFailures(), empty()); + assertThat(response.getTasks(), hasSize(1)); + TaskInfo task = response.getTasks().get(0); + assertEquals(TEST_TASK_ACTION.name(), task.action()); + }); } public void testGetTaskWaitForCompletionWithoutStoringResult() throws Exception { @@ -582,34 +594,20 @@ private void waitForCompletionTestCase(boolean storeResult, Function future = client().execute(TEST_TASK_ACTION, request); + ActionFuture future = ensureStartedOnAllNodes( + TEST_TASK_ACTION.name() + "[n]", + () -> client().execute(TEST_TASK_ACTION, request) + ); ActionFuture waitResponseFuture; - TaskId taskId; try { - taskId = waitForTestTaskStartOnAllNodes(); - - // Wait for the task to start - assertBusy(() -> clusterAdmin().prepareGetTask(taskId).get()); - - // Register listeners so we can be sure the waiting started - CountDownLatch waitForWaitingToStart = new CountDownLatch(1); - for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { - ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() { - @Override - public void onTaskUnregistered(Task task) { - waitForWaitingToStart.countDown(); - } - }); - } + var tasks = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).get().getTasks(); + assertThat(tasks, hasSize(1)); + var taskId = tasks.get(0).taskId(); + clusterAdmin().prepareGetTask(taskId).get(); // Spin up a request to wait for the test task to finish waitResponseFuture = wait.apply(taskId); - - /* Wait for the wait to start. This should count down just *before* we wait for completion but after the list/get has got a - * reference to the running task. Because we unblock immediately after this the task may no longer be running for us to wait - * on which is fine. */ - waitForWaitingToStart.await(); } finally { // Unblock the request so the wait for completion request can finish client().execute(UNBLOCK_TASK_ACTION, new TestTaskPlugin.UnblockTestTasksRequest()).get(); @@ -651,14 +649,15 @@ public void testGetTaskWaitForTimeout() throws Exception { */ private void waitForTimeoutTestCase(Function> wait) throws Exception { // Start blocking test task - TestTaskPlugin.NodesRequest request = new TestTaskPlugin.NodesRequest("test"); - ActionFuture future = client().execute(TEST_TASK_ACTION, request); + ActionFuture future = ensureStartedOnAllNodes( + TEST_TASK_ACTION.name() + "[n]", + () -> client().execute(TEST_TASK_ACTION, new TestTaskPlugin.NodesRequest("test")) + ); try { - TaskId taskId = waitForTestTaskStartOnAllNodes(); - - // Wait for the task to start - assertBusy(() -> clusterAdmin().prepareGetTask(taskId).get()); - + var tasks = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).get().getTasks(); + assertThat(tasks, hasSize(1)); + var taskId = tasks.get(0).taskId(); + clusterAdmin().prepareGetTask(taskId).get(); // Spin up a request that should wait for those tasks to finish // It will timeout because we haven't unblocked the tasks Iterable failures = wait.apply(taskId); @@ -675,17 +674,21 @@ private void waitForTimeoutTestCase(Function { - List tasks = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "[n]").get().getTasks(); - assertEquals(internalCluster().size(), tasks.size()); - }); - List task = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).get().getTasks(); - assertThat(task, hasSize(1)); - return task.get(0).taskId(); + private ActionFuture ensureStartedOnAllNodes(String nodeTaskName, Supplier> taskStarter) { + var startedOnAllNodes = new CountDownLatch(internalCluster().size()); + for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { + ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() { + @Override + public void onTaskRegistered(Task task) { + if (Objects.equals(task.getAction(), nodeTaskName)) { + startedOnAllNodes.countDown(); + } + } + }); + } + var future = taskStarter.get(); + safeAwait(startedOnAllNodes); + return future; } public void testTasksListWaitForNoTask() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java index efe3b097cae20..5a7f4609a7d0f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; @@ -24,6 +25,7 @@ import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING; @@ -45,7 +47,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { this.nodesInCluster = nodesInCluster; return List.of(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java index 02816688f1bbb..5a5fad9be3ef2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/UpdateHealthInfoCacheIT.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -77,7 +78,7 @@ public void testHealthNodeFailOver() throws Exception { DiscoveryNode newHealthNode = waitAndGetHealthNode(internalCluster); assertThat(newHealthNode, notNullValue()); logger.info("Previous health node {}, new health node {}.", healthNodeToBeShutDown, newHealthNode); - assertBusy(() -> assertResultsCanBeFetched(internalCluster, newHealthNode, List.of(nodeIds), null)); + assertBusy(() -> assertResultsCanBeFetched(internalCluster, newHealthNode, List.of(nodeIds), null), 30, TimeUnit.SECONDS); } @TestLogging(value = "org.elasticsearch.health.node:DEBUG", reason = "https://github.com/elastic/elasticsearch/issues/97213") @@ -93,7 +94,7 @@ public void testMasterFailure() throws Exception { ensureStableCluster(nodeIds.length); DiscoveryNode newHealthNode = waitAndGetHealthNode(internalCluster); assertThat(newHealthNode, notNullValue()); - assertBusy(() -> assertResultsCanBeFetched(internalCluster, newHealthNode, List.of(nodeIds), null)); + assertBusy(() -> assertResultsCanBeFetched(internalCluster, newHealthNode, List.of(nodeIds), null), 30, TimeUnit.SECONDS); } /** diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index 4c1c564bdc734..76d305ce8ea4b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -20,12 +20,15 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.field.WriteField; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,21 +38,29 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; public class DynamicMappingIT extends ESIntegTestCase { @@ -77,6 +88,16 @@ public void testConflictingDynamicMappings() { } } + public void testSimpleDynamicMappingsSuccessful() { + createIndex("index"); + client().prepareIndex("index").setId("1").setSource("a.x", 1).get(); + client().prepareIndex("index").setId("2").setSource("a.y", 2).get(); + + Map mappings = indicesAdmin().prepareGetMappings("index").get().mappings().get("index").sourceAsMap(); + assertTrue(new WriteField("properties.a", () -> mappings).exists()); + assertTrue(new WriteField("properties.a.properties.x", () -> mappings).exists()); + } + public void testConflictingDynamicMappingsBulk() { // we don't use indexRandom because the order of requests is important here createIndex("index"); @@ -87,18 +108,60 @@ public void testConflictingDynamicMappingsBulk() { assertTrue(bulkResponse.hasFailures()); } - private static void assertMappingsHaveField(GetMappingsResponse mappings, String index, String field) throws IOException { - MappingMetadata indexMappings = mappings.getMappings().get("index"); - assertNotNull(indexMappings); - Map typeMappingsMap = indexMappings.getSourceAsMap(); - @SuppressWarnings("unchecked") - Map properties = (Map) typeMappingsMap.get("properties"); - assertTrue("Could not find [" + field + "] in " + typeMappingsMap.toString(), properties.containsKey(field)); + public void testArrayWithDifferentTypes() { + createIndex("index"); + BulkResponse bulkResponse = client().prepareBulk() + .add(client().prepareIndex("index").setId("1").setSource("foo", List.of(42, "bar"))) + .get(); + + assertTrue(bulkResponse.hasFailures()); + assertEquals( + "mapper [foo] cannot be changed from type [long] to [text]", + bulkResponse.getItems()[0].getFailure().getCause().getMessage() + ); + } + + public void testArraysCountAsOneTowardsFieldLimit() { + createIndex("index", Settings.builder().put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2).build()); + BulkResponse bulkResponse = client().prepareBulk() + .add(client().prepareIndex("index").setId("1").setSource("field1", List.of(1, 2), "field2", 1)) + .get(); + + assertFalse(bulkResponse.hasFailures()); } public void testConcurrentDynamicUpdates() throws Throwable { - createIndex("index"); - final Thread[] indexThreads = new Thread[32]; + int numberOfFieldsToCreate = 32; + Map properties = indexConcurrently(numberOfFieldsToCreate, Settings.builder()); + assertThat(properties, aMapWithSize(numberOfFieldsToCreate)); + for (int i = 0; i < numberOfFieldsToCreate; i++) { + assertThat(properties, hasKey("field" + i)); + } + } + + public void testConcurrentDynamicIgnoreBeyondLimitUpdates() throws Throwable { + int numberOfFieldsToCreate = 32; + Map properties = indexConcurrently( + numberOfFieldsToCreate, + Settings.builder() + .put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfFieldsToCreate) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + ); + // every field is a multi-field (text + keyword) + assertThat(properties, aMapWithSize(16)); + assertResponse( + prepareSearch("index").setQuery(new MatchAllQueryBuilder()).setSize(numberOfFieldsToCreate).addFetchField("*"), + response -> { + long ignoredFields = Arrays.stream(response.getHits().getHits()).filter(hit -> hit.field("_ignored") != null).count(); + assertEquals(16, ignoredFields); + } + ); + } + + private Map indexConcurrently(int numberOfFieldsToCreate, Settings.Builder settings) throws Throwable { + indicesAdmin().prepareCreate("index").setSettings(settings).get(); + ensureGreen("index"); + final Thread[] indexThreads = new Thread[numberOfFieldsToCreate]; final CountDownLatch startLatch = new CountDownLatch(1); final AtomicReference error = new AtomicReference<>(); for (int i = 0; i < indexThreads.length; ++i) { @@ -126,14 +189,17 @@ public void run() { if (error.get() != null) { throw error.get(); } - Thread.sleep(2000); - GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get(); - for (int i = 0; i < indexThreads.length; ++i) { - assertMappingsHaveField(mappings, "index", "field" + i); - } - for (int i = 0; i < indexThreads.length; ++i) { + client().admin().indices().prepareRefresh("index").get(); + for (int i = 0; i < numberOfFieldsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } + GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get(); + MappingMetadata indexMappings = mappings.getMappings().get("index"); + assertNotNull(indexMappings); + Map typeMappingsMap = indexMappings.getSourceAsMap(); + @SuppressWarnings("unchecked") + Map properties = (Map) typeMappingsMap.get("properties"); + return properties; } public void testPreflightCheckAvoidsMaster() throws InterruptedException, IOException { @@ -221,15 +287,157 @@ public void onFailure(Exception e) { Exception e = expectThrows(DocumentParsingException.class, prepareIndex("index").setId("2").setSource("field2", "value2")); assertThat(e.getMessage(), Matchers.containsString("failed to parse")); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat( - e.getCause().getMessage(), - Matchers.containsString("Limit of total fields [2] has been exceeded while adding new fields [1]") - ); + assertThat(e.getCause().getMessage(), Matchers.containsString("Limit of total fields [2] has been exceeded")); } finally { indexingCompletedLatch.countDown(); } } + public void testIgnoreDynamicBeyondLimitSingleMultiField() { + indexIgnoreDynamicBeyond(1, orderedMap("field1", "text"), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("_ignored"))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field1"))); + }); + } + + public void testIgnoreDynamicBeyondLimitMultiField() { + indexIgnoreDynamicBeyond(2, orderedMap("field1", 1, "field2", "text"), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("field1", "_ignored"))); + assertThat(fields.get("field1").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field2"))); + }); + } + + public void testIgnoreDynamicArrayField() { + indexIgnoreDynamicBeyond(1, orderedMap("field1", 1, "field2", List.of(1, 2)), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("field1", "_ignored"))); + assertThat(fields.get("field1").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field2"))); + }); + } + + public void testIgnoreDynamicBeyondLimitObjectField() { + indexIgnoreDynamicBeyond(3, orderedMap("a.b", 1, "a.c", 2, "a.d", 3), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.c", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of(2L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("a.d"))); + }); + } + + public void testIgnoreDynamicBeyondLimitObjectField2() { + indexIgnoreDynamicBeyond(3, orderedMap("a.b", 1, "a.c", 2, "b.a", 3), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.c", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of(2L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("b"))); + }); + } + + public void testIgnoreDynamicBeyondLimitDottedObjectMultiField() { + indexIgnoreDynamicBeyond(4, orderedMap("a.b", "foo", "a.c", 2, "a.d", 3), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.b.keyword", "a.c", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.b.keyword").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of(2L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("a.d"))); + }); + } + + public void testIgnoreDynamicBeyondLimitObjectMultiField() { + indexIgnoreDynamicBeyond(5, orderedMap("a", orderedMap("b", "foo", "c", "bar", "d", 3)), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.b.keyword", "a.c", "a.c.keyword", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.b.keyword").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of("bar"))); + assertThat(fields.get("a.c.keyword").getValues(), equalTo(List.of("bar"))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("a.d"))); + }); + } + + public void testIgnoreDynamicBeyondLimitRuntimeFields() { + indexIgnoreDynamicBeyond(1, orderedMap("field1", 1, "field2", List.of(1, 2)), Map.of("dynamic", "runtime"), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("field1", "_ignored"))); + assertThat(fields.get("field1").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field2"))); + }); + } + + public void testFieldLimitRuntimeAndDynamic() throws Exception { + assertAcked( + indicesAdmin().prepareCreate("test") + .setSettings( + Settings.builder() + .put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 5) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + ) + .setMapping(""" + { + "dynamic": "runtime", + "properties": { + "runtime": { + "type": "object" + }, + "mapped_obj": { + "type": "object", + "dynamic": "true" + } + } + }""") + .get() + ); + + client().index( + new IndexRequest("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(orderedMap("dynamic.keyword", "foo", "mapped_obj.number", 1, "mapped_obj.string", "foo")) + ).get(); + + assertResponse(prepareSearch("test").setQuery(new MatchAllQueryBuilder()).addFetchField("*"), r -> { + var fields = r.getHits().getHits()[0].getFields(); + assertThat(fields.keySet(), equalTo(Set.of("dynamic.keyword", "mapped_obj.number", "_ignored"))); + assertThat(fields.get("dynamic.keyword").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("mapped_obj.number").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("mapped_obj.string"))); + }); + } + + private LinkedHashMap orderedMap(Object... entries) { + var map = new LinkedHashMap(); + for (int i = 0; i < entries.length; i += 2) { + map.put((String) entries[i], entries[i + 1]); + } + return map; + } + + private void indexIgnoreDynamicBeyond(int fieldLimit, Map source, Consumer> fieldsConsumer) { + indexIgnoreDynamicBeyond(fieldLimit, source, Map.of(), fieldsConsumer); + } + + private void indexIgnoreDynamicBeyond( + int fieldLimit, + Map source, + Map mapping, + Consumer> fieldsConsumer + ) { + client().admin() + .indices() + .prepareCreate("index") + .setSettings( + Settings.builder() + .put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), fieldLimit) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build() + ) + .setMapping(mapping) + .get(); + ensureGreen("index"); + client().prepareIndex("index").setId("1").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).setSource(source).get(); + assertResponse( + prepareSearch("index").setQuery(new MatchAllQueryBuilder()).addFetchField("*"), + r -> fieldsConsumer.accept(r.getHits().getHits()[0].getFields()) + ); + } + public void testTotalFieldsLimitWithRuntimeFields() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index bd400f9f0f6a1..1c4dbce2ccf32 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -1039,6 +1039,7 @@ public void testHistoryRetention() throws Exception { assertThat(recoveryState.getTranslog().recoveredOperations(), greaterThan(0)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105122") public void testDoNotInfinitelyWaitForMapping() { internalCluster().ensureAtLeastNumDataNodes(3); createIndex( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java index e85a7354f930d..809ecbc858706 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -26,6 +27,7 @@ import java.io.IOException; import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class RestControllerIT extends ESIntegTestCase { @@ -61,7 +63,8 @@ public Collection getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new BaseRestHandler() { @Override diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java index 5144aee654b31..c051bf6656bbc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/AggregationsIntegrationIT.java @@ -40,6 +40,7 @@ public void setupSuiteScopeCluster() throws Exception { public void testScroll() { final int size = randomIntBetween(1, 4); assertScrollResponsesAndHitCount( + client(), TimeValue.timeValueSeconds(60), prepareSearch("index").setSize(size).addAggregation(terms("f").field("f")), numDocs, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index dc512cdb92cc1..2ef951b28d3aa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -44,9 +43,8 @@ public void testDimensionFieldNameLimit() throws IOException { () -> List.of("routing_field"), dimensionFieldLimit ); - final Exception ex = expectThrows( - DocumentParsingException.class, - prepareIndex("test").setSource( + final DocWriteResponse response = client().prepareIndex("test") + .setSource( "routing_field", randomAlphaOfLength(10), dimensionFieldName, @@ -56,13 +54,8 @@ public void testDimensionFieldNameLimit() throws IOException { "@timestamp", Instant.now().toEpochMilli() ) - ); - assertThat( - ex.getCause().getMessage(), - equalTo( - "Dimension name must be less than [512] bytes but [" + dimensionFieldName + "] was [" + dimensionFieldName.length() + "]." - ) - ); + .get(); + assertEquals(RestStatus.CREATED.getStatus(), response.status().getStatus()); } public void testDimensionFieldValueLimit() throws IOException { @@ -74,20 +67,14 @@ public void testDimensionFieldValueLimit() throws IOException { dimensionFieldLimit ); long startTime = Instant.now().toEpochMilli(); - prepareIndex("test").setSource("field", randomAlphaOfLength(1024), "gauge", randomIntBetween(10, 20), "@timestamp", startTime) + final DocWriteResponse response1 = client().prepareIndex("test") + .setSource("field", randomAlphaOfLength(1024), "gauge", randomIntBetween(10, 20), "@timestamp", startTime) .get(); - final Exception ex = expectThrows( - DocumentParsingException.class, - prepareIndex("test").setSource( - "field", - randomAlphaOfLength(1025), - "gauge", - randomIntBetween(10, 20), - "@timestamp", - startTime + 1 - ) - ); - assertThat(ex.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1025].")); + final DocWriteResponse response2 = client().prepareIndex("test") + .setSource("field", randomAlphaOfLength(1025), "gauge", randomIntBetween(10, 20), "@timestamp", startTime + 1) + .get(); + assertEquals(RestStatus.CREATED.getStatus(), response1.status().getStatus()); + assertEquals(RestStatus.CREATED.getStatus(), response2.status().getStatus()); } public void testTotalNumberOfDimensionFieldsLimit() { @@ -107,7 +94,7 @@ public void testTotalNumberOfDimensionFieldsLimit() { } public void testTotalNumberOfDimensionFieldsDefaultLimit() { - int dimensionFieldLimit = 21; + int dimensionFieldLimit = 32 * 1024; final Exception ex = expectThrows(IllegalArgumentException.class, () -> createTimeSeriesIndex(mapping -> { mapping.startObject("routing_field").field("type", "keyword").field("time_series_dimension", true).endObject(); for (int i = 0; i < dimensionFieldLimit; i++) { @@ -169,8 +156,8 @@ public void testTotalDimensionFieldsSizeLuceneLimitPlusOne() throws IOException for (int i = 0; i < dimensionFieldLimit; i++) { source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1024)); } - final Exception ex = expectThrows(DocumentParsingException.class, prepareIndex("test").setSource(source)); - assertEquals("_tsid longer than [32766] bytes [33903].", ex.getCause().getMessage()); + final DocWriteResponse response = client().prepareIndex("test").setSource(source).get(); + assertEquals(RestStatus.CREATED.getStatus(), response.status().getStatus()); } private void createTimeSeriesIndex( @@ -188,6 +175,7 @@ private void createTimeSeriesIndex( Settings.Builder settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 38 * 1024) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPaths.get()) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-08T23:40:53.384Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z"); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index eddc96764273c..78086d28446b6 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -325,6 +325,7 @@ exports org.elasticsearch.search.aggregations; exports org.elasticsearch.search.aggregations.bucket; exports org.elasticsearch.search.aggregations.bucket.composite; + exports org.elasticsearch.search.aggregations.bucket.countedterms; exports org.elasticsearch.search.aggregations.bucket.filter; exports org.elasticsearch.search.aggregations.bucket.geogrid; exports org.elasticsearch.search.aggregations.bucket.global; @@ -423,6 +424,10 @@ org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat, org.elasticsearch.index.codec.postings.ES812PostingsFormat; provides org.apache.lucene.codecs.DocValuesFormat with ES87TSDBDocValuesFormat; + provides org.apache.lucene.codecs.KnnVectorsFormat + with + org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat, + org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat; exports org.elasticsearch.cluster.routing.allocation.shards to diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index 4f29fb3a168b3..656d213e7a1fd 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -37,6 +37,7 @@ import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.UnsupportedAggregationOnDownsampledIndex; +import org.elasticsearch.search.query.QueryPhaseTimeoutException; import org.elasticsearch.transport.TcpTransport; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentFragment; @@ -1896,6 +1897,12 @@ private enum ElasticsearchExceptionHandle { AutoscalingMissedIndicesUpdateException::new, 175, TransportVersions.MISSED_INDICES_UPDATE_EXCEPTION_ADDED + ), + QUERY_PHASE_TIMEOUT_EXCEPTION( + QueryPhaseTimeoutException.class, + QueryPhaseTimeoutException::new, + 176, + TransportVersions.QUERY_PHASE_TIMEOUT_EXCEPTION_ADDED ); final Class exceptionClass; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 6f0488a8848ea..cd7f9eb756b91 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -167,6 +167,10 @@ static TransportVersion def(int id) { public static final TransportVersion DESIRED_NODE_VERSION_OPTIONAL_STRING = def(8_580_00_0); public static final TransportVersion ML_INFERENCE_REQUEST_INPUT_TYPE_UNSPECIFIED_ADDED = def(8_581_00_0); public static final TransportVersion ASYNC_SEARCH_STATUS_SUPPORTS_KEEP_ALIVE = def(8_582_00_0); + public static final TransportVersion KNN_QUERY_NUMCANDS_AS_OPTIONAL_PARAM = def(8_583_00_0); + public static final TransportVersion TRANSFORM_GET_BASIC_STATS = def(8_584_00_0); + public static final TransportVersion NLP_DOCUMENT_CHUNKING_ADDED = def(8_585_00_0); + public static final TransportVersion QUERY_PHASE_TIMEOUT_EXCEPTION_ADDED = def(8_586_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 4d3079ae88465..6ee26e8fb7e4e 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -1017,7 +1017,8 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< indexScopedSettings, settingsFilter, indexNameExpressionResolver, - nodesInCluster + nodesInCluster, + clusterSupportsFeature )) { registerHandler.accept(handler); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java index 62ede5b2f480b..4f8a6b6db2980 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java @@ -8,8 +8,6 @@ package org.elasticsearch.action.admin.cluster.node.tasks.list; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; @@ -43,8 +41,6 @@ public class TransportListTasksAction extends TransportTasksAction { - private static final Logger logger = LogManager.getLogger(TransportListTasksAction.class); - public static final ActionType TYPE = new ActionType<>("cluster:monitor/tasks/lists"); public static long waitForCompletionTimeout(TimeValue timeout) { @@ -132,7 +128,6 @@ protected void processTasks(CancellableTask nodeTask, ListTasksRequest request, } processedTasks.add(task); } - logger.trace("Matched {} tasks of all running {}", processedTasks, taskManager.getTasks().values()); } catch (Exception e) { allMatchedTasksRemovedListener.onFailure(e); return; diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java new file mode 100644 index 0000000000000..1d95f430d5c7e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -0,0 +1,392 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.RoutingMissingException; +import org.elasticsearch.action.support.RefCountingRunnable; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndexClosedException; +import org.elasticsearch.node.NodeClosedException; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; + +import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY; + +/** + * retries on retryable cluster blocks, resolves item requests, + * constructs shard bulk requests and delegates execution to shard bulk action + */ +final class BulkOperation extends ActionRunnable { + + private static final Logger logger = LogManager.getLogger(BulkOperation.class); + + private final Task task; + private final ThreadPool threadPool; + private final ClusterService clusterService; + private BulkRequest bulkRequest; // set to null once all requests are sent out + private final ActionListener listener; + private final AtomicArray responses; + private final long startTimeNanos; + private final ClusterStateObserver observer; + private final Map indicesThatCannotBeCreated; + private final String executorName; + private final LongSupplier relativeTimeProvider; + private IndexNameExpressionResolver indexNameExpressionResolver; + private NodeClient client; + + BulkOperation( + Task task, + ThreadPool threadPool, + String executorName, + ClusterService clusterService, + BulkRequest bulkRequest, + NodeClient client, + AtomicArray responses, + Map indicesThatCannotBeCreated, + IndexNameExpressionResolver indexNameExpressionResolver, + LongSupplier relativeTimeProvider, + long startTimeNanos, + ActionListener listener + ) { + super(listener); + this.task = task; + this.threadPool = threadPool; + this.clusterService = clusterService; + this.responses = responses; + this.bulkRequest = bulkRequest; + this.listener = listener; + this.startTimeNanos = startTimeNanos; + this.indicesThatCannotBeCreated = indicesThatCannotBeCreated; + this.executorName = executorName; + this.relativeTimeProvider = relativeTimeProvider; + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.client = client; + this.observer = new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext()); + } + + @Override + protected void doRun() { + assert bulkRequest != null; + final ClusterState clusterState = observer.setAndGetObservedState(); + if (handleBlockExceptions(clusterState)) { + return; + } + Map> requestsByShard = groupRequestsByShards(clusterState); + executeBulkRequestsByShard(requestsByShard, clusterState); + } + + private long buildTookInMillis(long startTimeNanos) { + return TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - startTimeNanos); + } + + private Map> groupRequestsByShards(ClusterState clusterState) { + final ConcreteIndices concreteIndices = new ConcreteIndices(clusterState, indexNameExpressionResolver); + Metadata metadata = clusterState.metadata(); + // Group the requests by ShardId -> Operations mapping + Map> requestsByShard = new HashMap<>(); + + for (int i = 0; i < bulkRequest.requests.size(); i++) { + DocWriteRequest docWriteRequest = bulkRequest.requests.get(i); + // the request can only be null because we set it to null in the previous step, so it gets ignored + if (docWriteRequest == null) { + continue; + } + if (addFailureIfRequiresAliasAndAliasIsMissing(docWriteRequest, i, metadata)) { + continue; + } + if (addFailureIfIndexCannotBeCreated(docWriteRequest, i)) { + continue; + } + if (addFailureIfRequiresDataStreamAndNoParentDataStream(docWriteRequest, i, metadata)) { + continue; + } + IndexAbstraction ia = null; + boolean includeDataStreams = docWriteRequest.opType() == DocWriteRequest.OpType.CREATE; + try { + ia = concreteIndices.resolveIfAbsent(docWriteRequest); + if (ia.isDataStreamRelated() && includeDataStreams == false) { + throw new IllegalArgumentException("only write ops with an op_type of create are allowed in data streams"); + } + // The ConcreteIndices#resolveIfAbsent(...) method validates via IndexNameExpressionResolver whether + // an operation is allowed in index into a data stream, but this isn't done when resolve call is cached, so + // the validation needs to be performed here too. + if (ia.getParentDataStream() != null && + // avoid valid cases when directly indexing into a backing index + // (for example when directly indexing into .ds-logs-foobar-000001) + ia.getName().equals(docWriteRequest.index()) == false && docWriteRequest.opType() != DocWriteRequest.OpType.CREATE) { + throw new IllegalArgumentException("only write ops with an op_type of create are allowed in data streams"); + } + + TransportBulkAction.prohibitCustomRoutingOnDataStream(docWriteRequest, metadata); + TransportBulkAction.prohibitAppendWritesInBackingIndices(docWriteRequest, metadata); + docWriteRequest.routing(metadata.resolveWriteIndexRouting(docWriteRequest.routing(), docWriteRequest.index())); + + final Index concreteIndex = docWriteRequest.getConcreteWriteIndex(ia, metadata); + if (addFailureIfIndexIsClosed(docWriteRequest, concreteIndex, i, metadata)) { + continue; + } + IndexRouting indexRouting = concreteIndices.routing(concreteIndex); + docWriteRequest.process(indexRouting); + int shardId = docWriteRequest.route(indexRouting); + List shardRequests = requestsByShard.computeIfAbsent( + new ShardId(concreteIndex, shardId), + shard -> new ArrayList<>() + ); + shardRequests.add(new BulkItemRequest(i, docWriteRequest)); + } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException | ResourceNotFoundException e) { + String name = ia != null ? ia.getName() : docWriteRequest.index(); + BulkItemResponse.Failure failure = new BulkItemResponse.Failure(name, docWriteRequest.id(), e); + BulkItemResponse bulkItemResponse = BulkItemResponse.failure(i, docWriteRequest.opType(), failure); + responses.set(i, bulkItemResponse); + // make sure the request gets never processed again + bulkRequest.requests.set(i, null); + } + } + return requestsByShard; + } + + private void executeBulkRequestsByShard(Map> requestsByShard, ClusterState clusterState) { + if (requestsByShard.isEmpty()) { + listener.onResponse( + new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)) + ); + return; + } + + String nodeId = clusterService.localNode().getId(); + Runnable onBulkItemsComplete = () -> { + listener.onResponse( + new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)) + ); + // Allow memory for bulk shard request items to be reclaimed before all items have been completed + bulkRequest = null; + }; + + try (RefCountingRunnable bulkItemRequestCompleteRefCount = new RefCountingRunnable(onBulkItemsComplete)) { + for (Map.Entry> entry : requestsByShard.entrySet()) { + final ShardId shardId = entry.getKey(); + final List requests = entry.getValue(); + + BulkShardRequest bulkShardRequest = new BulkShardRequest( + shardId, + bulkRequest.getRefreshPolicy(), + requests.toArray(new BulkItemRequest[0]) + ); + bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); + bulkShardRequest.timeout(bulkRequest.timeout()); + bulkShardRequest.routedBasedOnClusterVersion(clusterState.version()); + if (task != null) { + bulkShardRequest.setParentTask(nodeId, task.getId()); + } + executeBulkShardRequest(bulkShardRequest, bulkItemRequestCompleteRefCount.acquire()); + } + } + } + + private void executeBulkShardRequest(BulkShardRequest bulkShardRequest, Releasable releaseOnFinish) { + client.executeLocally(TransportShardBulkAction.TYPE, bulkShardRequest, new ActionListener<>() { + @Override + public void onResponse(BulkShardResponse bulkShardResponse) { + for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) { + // we may have no response if item failed + if (bulkItemResponse.getResponse() != null) { + bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); + } + responses.set(bulkItemResponse.getItemId(), bulkItemResponse); + } + releaseOnFinish.close(); + } + + @Override + public void onFailure(Exception e) { + // create failures for all relevant requests + for (BulkItemRequest request : bulkShardRequest.items()) { + final String indexName = request.index(); + DocWriteRequest docWriteRequest = request.request(); + BulkItemResponse.Failure failure = new BulkItemResponse.Failure(indexName, docWriteRequest.id(), e); + responses.set(request.id(), BulkItemResponse.failure(request.id(), docWriteRequest.opType(), failure)); + } + releaseOnFinish.close(); + } + }); + } + + private boolean handleBlockExceptions(ClusterState state) { + ClusterBlockException blockException = state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); + if (blockException != null) { + if (blockException.retryable()) { + logger.trace("cluster is blocked, scheduling a retry", blockException); + retry(blockException); + } else { + onFailure(blockException); + } + return true; + } + return false; + } + + void retry(Exception failure) { + assert failure != null; + if (observer.isTimedOut()) { + // we running as a last attempt after a timeout has happened. don't retry + onFailure(failure); + return; + } + observer.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + /* + * This is called on the cluster state update thread pool + * but we'd prefer to coordinate the bulk request on the + * write thread pool just to make sure the cluster state + * update thread doesn't get clogged up. + */ + dispatchRetry(); + } + + @Override + public void onClusterServiceClose() { + onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + /* + * Try one more time.... This is called on the generic + * thread pool but out of an abundance of caution we + * switch over to the write thread pool that we expect + * to coordinate the bulk request. + */ + dispatchRetry(); + } + + private void dispatchRetry() { + threadPool.executor(executorName).submit(BulkOperation.this); + } + }); + } + + private boolean addFailureIfRequiresAliasAndAliasIsMissing(DocWriteRequest request, int idx, final Metadata metadata) { + if (request.isRequireAlias() && (metadata.hasAlias(request.index()) == false)) { + Exception exception = new IndexNotFoundException( + "[" + DocWriteRequest.REQUIRE_ALIAS + "] request flag is [true] and [" + request.index() + "] is not an alias", + request.index() + ); + addFailure(request, idx, exception); + return true; + } + return false; + } + + private boolean addFailureIfRequiresDataStreamAndNoParentDataStream(DocWriteRequest request, int idx, final Metadata metadata) { + if (request.isRequireDataStream() && (metadata.indexIsADataStream(request.index()) == false)) { + Exception exception = new ResourceNotFoundException( + "[" + DocWriteRequest.REQUIRE_DATA_STREAM + "] request flag is [true] and [" + request.index() + "] is not a data stream", + request.index() + ); + addFailure(request, idx, exception); + return true; + } + return false; + } + + private boolean addFailureIfIndexIsClosed(DocWriteRequest request, Index concreteIndex, int idx, final Metadata metadata) { + IndexMetadata indexMetadata = metadata.getIndexSafe(concreteIndex); + if (indexMetadata.getState() == IndexMetadata.State.CLOSE) { + addFailure(request, idx, new IndexClosedException(concreteIndex)); + return true; + } + return false; + } + + private boolean addFailureIfIndexCannotBeCreated(DocWriteRequest request, int idx) { + IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index()); + if (cannotCreate != null) { + addFailure(request, idx, cannotCreate); + return true; + } + return false; + } + + private void addFailure(DocWriteRequest request, int idx, Exception unavailableException) { + BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.id(), unavailableException); + BulkItemResponse bulkItemResponse = BulkItemResponse.failure(idx, request.opType(), failure); + responses.set(idx, bulkItemResponse); + // make sure the request gets never processed again + bulkRequest.requests.set(idx, null); + } + + private static class ConcreteIndices { + private final ClusterState state; + private final IndexNameExpressionResolver indexNameExpressionResolver; + private final Map indexAbstractions = new HashMap<>(); + private final Map routings = new HashMap<>(); + + ConcreteIndices(ClusterState state, IndexNameExpressionResolver indexNameExpressionResolver) { + this.state = state; + this.indexNameExpressionResolver = indexNameExpressionResolver; + } + + IndexAbstraction resolveIfAbsent(DocWriteRequest request) { + try { + IndexAbstraction indexAbstraction = indexAbstractions.get(request.index()); + if (indexAbstraction == null) { + indexAbstraction = indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request); + indexAbstractions.put(request.index(), indexAbstraction); + } + return indexAbstraction; + } catch (IndexNotFoundException e) { + if (e.getMetadataKeys().contains(EXCLUDED_DATA_STREAMS_KEY)) { + throw new IllegalArgumentException("only write ops with an op_type of create are allowed in data streams", e); + } else { + throw e; + } + } + } + + IndexRouting routing(Index index) { + IndexRouting routing = routings.get(index); + if (routing == null) { + routing = IndexRouting.fromIndexMetadata(state.metadata().getIndexSafe(index)); + routings.put(index, routing); + } + return routing; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index 16e5430063650..24d6fad554935 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -8,11 +8,9 @@ package org.elasticsearch.action.bulk; -import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestLazyBuilder; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.RequestBuilder; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.index.IndexRequest; @@ -46,14 +44,13 @@ public class BulkRequestBuilder extends ActionRequestLazyBuilder> requests = new ArrayList<>(); private final List framedData = new ArrayList<>(); - private final List> requestBuilders = new ArrayList<>(); + private final List, ? extends DocWriteResponse>> requestBuilders = + new ArrayList<>(); private ActiveShardCount waitForActiveShards; private TimeValue timeout; - private String timeoutString; private String globalPipeline; private String globalRouting; private WriteRequest.RefreshPolicy refreshPolicy; - private String refreshPolicyString; public BulkRequestBuilder(ElasticsearchClient client, @Nullable String globalIndex) { super(client, BulkAction.INSTANCE); @@ -167,7 +164,7 @@ public final BulkRequestBuilder setTimeout(TimeValue timeout) { * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}. */ public final BulkRequestBuilder setTimeout(String timeout) { - this.timeoutString = timeout; + this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"); return this; } @@ -196,7 +193,7 @@ public BulkRequestBuilder setRefreshPolicy(WriteRequest.RefreshPolicy refreshPol @Override public BulkRequestBuilder setRefreshPolicy(String refreshPolicy) { - this.refreshPolicyString = refreshPolicy; + this.refreshPolicy = WriteRequest.RefreshPolicy.parse(refreshPolicy); return this; } @@ -204,9 +201,9 @@ public BulkRequestBuilder setRefreshPolicy(String refreshPolicy) { public BulkRequest request() { validate(); BulkRequest request = new BulkRequest(globalIndex); - for (RequestBuilder requestBuilder : requestBuilders) { - ActionRequest childRequest = requestBuilder.request(); - request.add((DocWriteRequest) childRequest); + for (ActionRequestLazyBuilder, ? extends DocWriteResponse> requestBuilder : requestBuilders) { + DocWriteRequest childRequest = requestBuilder.request(); + request.add(childRequest); } for (DocWriteRequest childRequest : requests) { request.add(childRequest); @@ -224,9 +221,6 @@ public BulkRequest request() { if (timeout != null) { request.timeout(timeout); } - if (timeoutString != null) { - request.timeout(timeoutString); - } if (globalPipeline != null) { request.pipeline(globalPipeline); } @@ -236,9 +230,6 @@ public BulkRequest request() { if (refreshPolicy != null) { request.setRefreshPolicy(refreshPolicy); } - if (refreshPolicyString != null) { - request.setRefreshPolicy(refreshPolicyString); - } return request; } @@ -248,12 +239,6 @@ private void validate() { "Must use only request builders, requests, or byte arrays within a single bulk request. Cannot mix and match" ); } - if (timeout != null && timeoutString != null) { - throw new IllegalStateException("Must use only one setTimeout method"); - } - if (refreshPolicy != null && refreshPolicyString != null) { - throw new IllegalStateException("Must use only one setRefreshPolicy method"); - } } private int countNonEmptyLists(List... lists) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java new file mode 100644 index 0000000000000..5e630bf9cdef5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestModifier.java @@ -0,0 +1,270 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SparseFixedBitSet; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.Assertions; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.ingest.IngestService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; + +/** + * Manages mutations to a bulk request that arise from the application of ingest pipelines. The modifier acts as an iterator over the + * documents of a bulk request, keeping a record of all dropped and failed write requests in the overall bulk operation. + * Once all pipelines have been applied, the modifier is used to create a new bulk request that will be used for executing the + * remaining writes. When this final bulk operation is completed, the modifier is used to combine the results with those from the + * ingest service to create the final bulk response. + */ +final class BulkRequestModifier implements Iterator> { + + private static final Logger logger = LogManager.getLogger(BulkRequestModifier.class); + + private static final String DROPPED_OR_FAILED_ITEM_WITH_AUTO_GENERATED_ID = "auto-generated"; + + final BulkRequest bulkRequest; + final SparseFixedBitSet failedSlots; + final List itemResponses; + final AtomicIntegerArray originalSlots; + + volatile int currentSlot = -1; + + BulkRequestModifier(BulkRequest bulkRequest) { + this.bulkRequest = bulkRequest; + this.failedSlots = new SparseFixedBitSet(bulkRequest.requests().size()); + this.itemResponses = new ArrayList<>(bulkRequest.requests().size()); + this.originalSlots = new AtomicIntegerArray(bulkRequest.requests().size()); // oversize, but that's ok + } + + @Override + public DocWriteRequest next() { + return bulkRequest.requests().get(++currentSlot); + } + + @Override + public boolean hasNext() { + return (currentSlot + 1) < bulkRequest.requests().size(); + } + + /** + * Creates a new bulk request containing all documents from the original bulk request that have not been marked as failed + * or dropped. Any failed or dropped documents are tracked as a side effect of this call so that they may be reflected in the + * final bulk response. + * + * @return A new bulk request without the write operations removed during any ingest pipeline executions. + */ + BulkRequest getBulkRequest() { + if (itemResponses.isEmpty()) { + return bulkRequest; + } else { + BulkRequest modifiedBulkRequest = new BulkRequest(); + modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); + modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); + modifiedBulkRequest.timeout(bulkRequest.timeout()); + + int slot = 0; + List> requests = bulkRequest.requests(); + for (int i = 0; i < requests.size(); i++) { + DocWriteRequest request = requests.get(i); + if (failedSlots.get(i) == false) { + modifiedBulkRequest.add(request); + originalSlots.set(slot++, i); + } + } + return modifiedBulkRequest; + } + } + + /** + * If documents were dropped or failed in ingest, this method wraps the action listener that will be notified when the + * updated bulk operation is completed. The wrapped listener combines the dropped and failed document results from the ingest + * service with the results returned from running the remaining write operations. + * + * @param ingestTookInMillis Time elapsed for ingestion to be passed to final result. + * @param actionListener The action listener that expects the final bulk response. + * @return An action listener that combines ingest failure results with the results from writing the remaining documents. + */ + ActionListener wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener actionListener) { + if (itemResponses.isEmpty()) { + return actionListener.map( + response -> new BulkResponse(response.getItems(), response.getTook().getMillis(), ingestTookInMillis) + ); + } else { + return actionListener.map(response -> { + // these items are the responses from the subsequent bulk request, their 'slots' + // are not correct for this response we're building + final BulkItemResponse[] bulkResponses = response.getItems(); + + final BulkItemResponse[] allResponses = new BulkItemResponse[bulkResponses.length + itemResponses.size()]; + + // the item responses are from the original request, so their slots are correct. + // these are the responses for requests that failed early and were not passed on to the subsequent bulk. + for (BulkItemResponse item : itemResponses) { + allResponses[item.getItemId()] = item; + } + + // use the original slots for the responses from the bulk + for (int i = 0; i < bulkResponses.length; i++) { + allResponses[originalSlots.get(i)] = bulkResponses[i]; + } + + if (Assertions.ENABLED) { + assertResponsesAreCorrect(bulkResponses, allResponses); + } + + return new BulkResponse(allResponses, response.getTook().getMillis(), ingestTookInMillis); + }); + } + } + + private void assertResponsesAreCorrect(BulkItemResponse[] bulkResponses, BulkItemResponse[] allResponses) { + // check for an empty intersection between the ids + final Set failedIds = itemResponses.stream().map(BulkItemResponse::getItemId).collect(Collectors.toSet()); + final Set responseIds = IntStream.range(0, bulkResponses.length) + .map(originalSlots::get) // resolve subsequent bulk ids back to the original slots + .boxed() + .collect(Collectors.toSet()); + assert Sets.haveEmptyIntersection(failedIds, responseIds) + : "bulk item response slots cannot have failed and been processed in the subsequent bulk request, failed ids: " + + failedIds + + ", response ids: " + + responseIds; + + // check for the correct number of responses + final int expectedResponseCount = bulkRequest.requests.size(); + final int actualResponseCount = failedIds.size() + responseIds.size(); + assert expectedResponseCount == actualResponseCount + : "Expected [" + expectedResponseCount + "] responses, but found [" + actualResponseCount + "]"; + + // check that every response is present + for (int i = 0; i < allResponses.length; i++) { + assert allResponses[i] != null : "BulkItemResponse at index [" + i + "] was null"; + } + } + + /** + * Mark the document at the given slot in the bulk request as having failed in the ingest service. + * @param slot the slot in the bulk request to mark as failed. + * @param e the failure encountered. + */ + synchronized void markItemAsFailed(int slot, Exception e) { + final DocWriteRequest docWriteRequest = bulkRequest.requests().get(slot); + final String id = Objects.requireNonNullElse(docWriteRequest.id(), DROPPED_OR_FAILED_ITEM_WITH_AUTO_GENERATED_ID); + // We hit a error during preprocessing a request, so we: + // 1) Remember the request item slot from the bulk, so that when we're done processing all requests we know what failed + // 2) Add a bulk item failure for this request + // 3) Continue with the next request in the bulk. + failedSlots.set(slot); + BulkItemResponse.Failure failure = new BulkItemResponse.Failure(docWriteRequest.index(), id, e); + itemResponses.add(BulkItemResponse.failure(slot, docWriteRequest.opType(), failure)); + } + + /** + * Mark the document at the given slot in the bulk request as having been dropped by the ingest service. + * @param slot the slot in the bulk request to mark as dropped. + */ + synchronized void markItemAsDropped(int slot) { + final DocWriteRequest docWriteRequest = bulkRequest.requests().get(slot); + final String id = Objects.requireNonNullElse(docWriteRequest.id(), DROPPED_OR_FAILED_ITEM_WITH_AUTO_GENERATED_ID); + failedSlots.set(slot); + UpdateResponse dropped = new UpdateResponse( + new ShardId(docWriteRequest.index(), IndexMetadata.INDEX_UUID_NA_VALUE, 0), + id, + UNASSIGNED_SEQ_NO, + UNASSIGNED_PRIMARY_TERM, + docWriteRequest.version(), + DocWriteResponse.Result.NOOP + ); + itemResponses.add(BulkItemResponse.success(slot, docWriteRequest.opType(), dropped)); + } + + /** + * Mark the document at the given slot in the bulk request as having failed in the ingest service. The document will be redirected + * to a data stream's failure store. + * @param slot the slot in the bulk request to redirect. + * @param targetIndexName the index that the document was targeting at the time of failure. + * @param e the failure encountered. + */ + public void markItemForFailureStore(int slot, String targetIndexName, Exception e) { + if (DataStream.isFailureStoreEnabled() == false) { + // Assert false for development, but if we somehow find ourselves here, default to failure logic. + assert false + : "Attempting to route a failed write request type to a failure store but the failure store is not enabled! " + + "This should be guarded against in TransportBulkAction#shouldStoreFailure()"; + markItemAsFailed(slot, e); + } else { + // We get the index write request to find the source of the failed document + IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(bulkRequest.requests().get(slot)); + if (indexRequest == null) { + // This is unlikely to happen ever since only source oriented operations (index, create, upsert) are considered for + // ingest, but if it does happen, attempt to trip an assertion. If running in production, be defensive: Mark it failed + // as normal, and log the info for later debugging if needed. + assert false + : "Attempting to mark invalid write request type for failure store. Only IndexRequest or UpdateRequest allowed. " + + "type: [" + + bulkRequest.requests().get(slot).getClass().getName() + + "], index: [" + + targetIndexName + + "]"; + markItemAsFailed(slot, e); + logger.debug( + () -> "Attempted to redirect an invalid write operation after ingest failure - type: [" + + bulkRequest.requests().get(slot).getClass().getName() + + "], index: [" + + targetIndexName + + "]" + ); + } else { + try { + IndexRequest errorDocument = FailureStoreDocument.transformFailedRequest(indexRequest, e, targetIndexName); + // This is a fresh index request! We need to do some preprocessing on it. If we do not, when this is returned to + // the bulk action, the action will see that it hasn't been processed by ingest yet and attempt to ingest it again. + errorDocument.isPipelineResolved(true); + errorDocument.setPipeline(IngestService.NOOP_PIPELINE_NAME); + errorDocument.setFinalPipeline(IngestService.NOOP_PIPELINE_NAME); + bulkRequest.requests.set(slot, errorDocument); + } catch (IOException ioException) { + // This is unlikely to happen because the conversion is so simple, but be defensive and attempt to report about it + // if we need the info later. + e.addSuppressed(ioException); // Prefer to return the original exception to the end user instead of this new one. + logger.debug( + () -> "Encountered exception while attempting to redirect a failed ingest operation: index [" + + targetIndexName + + "], source: [" + + indexRequest.source().utf8ToString() + + "]", + ioException + ); + markItemAsFailed(slot, e); + } + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocument.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocument.java new file mode 100644 index 0000000000000..e0d6e8200e86d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocument.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Transforms an indexing request using error information into a new index request to be stored in a data stream's failure store. + */ +public final class FailureStoreDocument { + + private FailureStoreDocument() {} + + /** + * Combines an {@link IndexRequest} that has failed during the bulk process with the error thrown for that request. The result is a + * new {@link IndexRequest} that can be stored in a data stream's failure store. + * @param source The original request that has failed to be ingested + * @param exception The exception that was thrown that caused the request to fail to be ingested + * @param targetIndexName The index that the request was targeting at time of failure + * @return A new {@link IndexRequest} with a failure store compliant structure + * @throws IOException If there is a problem when the document's new source is serialized + */ + public static IndexRequest transformFailedRequest(IndexRequest source, Exception exception, String targetIndexName) throws IOException { + return transformFailedRequest(source, exception, targetIndexName, System::currentTimeMillis); + } + + /** + * Combines an {@link IndexRequest} that has failed during the bulk process with the error thrown for that request. The result is a + * new {@link IndexRequest} that can be stored in a data stream's failure store. + * @param source The original request that has failed to be ingested + * @param exception The exception that was thrown that caused the request to fail to be ingested + * @param targetIndexName The index that the request was targeting at time of failure + * @param timeSupplier Supplies the value for the document's timestamp + * @return A new {@link IndexRequest} with a failure store compliant structure + * @throws IOException If there is a problem when the document's new source is serialized + */ + public static IndexRequest transformFailedRequest( + IndexRequest source, + Exception exception, + String targetIndexName, + Supplier timeSupplier + ) throws IOException { + return new IndexRequest().index(targetIndexName) + .source(createSource(source, exception, targetIndexName, timeSupplier)) + .opType(DocWriteRequest.OpType.CREATE) + .setWriteToFailureStore(true); + } + + private static XContentBuilder createSource( + IndexRequest source, + Exception exception, + String targetIndexName, + Supplier timeSupplier + ) throws IOException { + Objects.requireNonNull(source, "source must not be null"); + Objects.requireNonNull(exception, "exception must not be null"); + Objects.requireNonNull(targetIndexName, "targetIndexName must not be null"); + Objects.requireNonNull(timeSupplier, "timeSupplier must not be null"); + Throwable unwrapped = ExceptionsHelper.unwrapCause(exception); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + { + builder.timeField("@timestamp", timeSupplier.get()); + builder.startObject("document"); + { + if (source.id() != null) { + builder.field("id", source.id()); + } + if (source.routing() != null) { + builder.field("routing", source.routing()); + } + builder.field("index", source.index()); + // Unmapped source field + builder.startObject("source"); + { + builder.mapContents(source.sourceAsMap()); + } + builder.endObject(); + } + builder.endObject(); + builder.startObject("error"); + { + builder.field("type", ElasticsearchException.getExceptionName(unwrapped)); + builder.field("message", unwrapped.getMessage()); + builder.field("stack_trace", ExceptionsHelper.stackTrace(unwrapped)); + // Further fields not yet tracked (Need to expose via specific exceptions) + // - pipeline + // - pipeline_trace + // - processor + } + builder.endObject(); + } + builder.endObject(); + return builder; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index de11a57a237df..2f12008501487 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -10,18 +10,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.SparseFixedBitSet; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteRequest.OpType; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -36,24 +31,22 @@ import org.elasticsearch.action.support.WriteResponse; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; @@ -61,8 +54,6 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.NodeClosedException; @@ -71,21 +62,16 @@ import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.TransportService; -import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.SortedMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.function.LongSupplier; import java.util.stream.Collectors; -import java.util.stream.IntStream; -import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -105,7 +91,6 @@ public class TransportBulkAction extends HandledTransportAction { - private final Task task; - private BulkRequest bulkRequest; // set to null once all requests are sent out - private final ActionListener listener; - private final AtomicArray responses; - private final long startTimeNanos; - private final ClusterStateObserver observer; - private final Map indicesThatCannotBeCreated; - private final String executorName; - - BulkOperation( - Task task, - BulkRequest bulkRequest, - ActionListener listener, - String executorName, - AtomicArray responses, - long startTimeNanos, - Map indicesThatCannotBeCreated - ) { - super(listener); - this.task = task; - this.bulkRequest = bulkRequest; - this.listener = listener; - this.responses = responses; - this.startTimeNanos = startTimeNanos; - this.indicesThatCannotBeCreated = indicesThatCannotBeCreated; - this.executorName = executorName; - this.observer = new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext()); - } - - @Override - protected void doRun() { - assert bulkRequest != null; - final ClusterState clusterState = observer.setAndGetObservedState(); - if (handleBlockExceptions(clusterState)) { - return; - } - Map> requestsByShard = groupRequestsByShards(clusterState); - executeBulkRequestsByShard(requestsByShard, clusterState); - } - - private Map> groupRequestsByShards(ClusterState clusterState) { - final ConcreteIndices concreteIndices = new ConcreteIndices(clusterState, indexNameExpressionResolver); - Metadata metadata = clusterState.metadata(); - // Group the requests by ShardId -> Operations mapping - Map> requestsByShard = new HashMap<>(); - - for (int i = 0; i < bulkRequest.requests.size(); i++) { - DocWriteRequest docWriteRequest = bulkRequest.requests.get(i); - // the request can only be null because we set it to null in the previous step, so it gets ignored - if (docWriteRequest == null) { - continue; - } - if (addFailureIfRequiresAliasAndAliasIsMissing(docWriteRequest, i, metadata)) { - continue; - } - if (addFailureIfIndexCannotBeCreated(docWriteRequest, i)) { - continue; - } - if (addFailureIfRequiresDataStreamAndNoParentDataStream(docWriteRequest, i, metadata)) { - continue; - } - IndexAbstraction ia = null; - boolean includeDataStreams = docWriteRequest.opType() == OpType.CREATE; - try { - ia = concreteIndices.resolveIfAbsent(docWriteRequest); - if (ia.isDataStreamRelated() && includeDataStreams == false) { - throw new IllegalArgumentException("only write ops with an op_type of create are allowed in data streams"); - } - // The ConcreteIndices#resolveIfAbsent(...) method validates via IndexNameExpressionResolver whether - // an operation is allowed in index into a data stream, but this isn't done when resolve call is cached, so - // the validation needs to be performed here too. - if (ia.getParentDataStream() != null && - // avoid valid cases when directly indexing into a backing index - // (for example when directly indexing into .ds-logs-foobar-000001) - ia.getName().equals(docWriteRequest.index()) == false && docWriteRequest.opType() != OpType.CREATE) { - throw new IllegalArgumentException("only write ops with an op_type of create are allowed in data streams"); - } - - prohibitCustomRoutingOnDataStream(docWriteRequest, metadata); - prohibitAppendWritesInBackingIndices(docWriteRequest, metadata); - docWriteRequest.routing(metadata.resolveWriteIndexRouting(docWriteRequest.routing(), docWriteRequest.index())); - - final Index concreteIndex = docWriteRequest.getConcreteWriteIndex(ia, metadata); - if (addFailureIfIndexIsClosed(docWriteRequest, concreteIndex, i, metadata)) { - continue; - } - IndexRouting indexRouting = concreteIndices.routing(concreteIndex); - docWriteRequest.process(indexRouting); - int shardId = docWriteRequest.route(indexRouting); - List shardRequests = requestsByShard.computeIfAbsent( - new ShardId(concreteIndex, shardId), - shard -> new ArrayList<>() - ); - shardRequests.add(new BulkItemRequest(i, docWriteRequest)); - } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException | ResourceNotFoundException e) { - String name = ia != null ? ia.getName() : docWriteRequest.index(); - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(name, docWriteRequest.id(), e); - BulkItemResponse bulkItemResponse = BulkItemResponse.failure(i, docWriteRequest.opType(), failure); - responses.set(i, bulkItemResponse); - // make sure the request gets never processed again - bulkRequest.requests.set(i, null); - } - } - return requestsByShard; - } - - private void executeBulkRequestsByShard(Map> requestsByShard, ClusterState clusterState) { - if (requestsByShard.isEmpty()) { - listener.onResponse( - new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)) - ); - return; - } - - String nodeId = clusterService.localNode().getId(); - Runnable onBulkItemsComplete = () -> { - listener.onResponse( - new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)) - ); - // Allow memory for bulk shard request items to be reclaimed before all items have been completed - bulkRequest = null; - }; - - try (RefCountingRunnable bulkItemRequestCompleteRefCount = new RefCountingRunnable(onBulkItemsComplete)) { - for (Map.Entry> entry : requestsByShard.entrySet()) { - final ShardId shardId = entry.getKey(); - final List requests = entry.getValue(); - - BulkShardRequest bulkShardRequest = new BulkShardRequest( - shardId, - bulkRequest.getRefreshPolicy(), - requests.toArray(new BulkItemRequest[0]) - ); - bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); - bulkShardRequest.timeout(bulkRequest.timeout()); - bulkShardRequest.routedBasedOnClusterVersion(clusterState.version()); - if (task != null) { - bulkShardRequest.setParentTask(nodeId, task.getId()); - } - executeBulkShardRequest(bulkShardRequest, bulkItemRequestCompleteRefCount.acquire()); - } - } - } - - private void executeBulkShardRequest(BulkShardRequest bulkShardRequest, Releasable releaseOnFinish) { - client.executeLocally(TransportShardBulkAction.TYPE, bulkShardRequest, new ActionListener<>() { - @Override - public void onResponse(BulkShardResponse bulkShardResponse) { - for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) { - // we may have no response if item failed - if (bulkItemResponse.getResponse() != null) { - bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); - } - responses.set(bulkItemResponse.getItemId(), bulkItemResponse); - } - releaseOnFinish.close(); - } - - @Override - public void onFailure(Exception e) { - // create failures for all relevant requests - for (BulkItemRequest request : bulkShardRequest.items()) { - final String indexName = request.index(); - DocWriteRequest docWriteRequest = request.request(); - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(indexName, docWriteRequest.id(), e); - responses.set(request.id(), BulkItemResponse.failure(request.id(), docWriteRequest.opType(), failure)); - } - releaseOnFinish.close(); - } - }); - } - - private boolean handleBlockExceptions(ClusterState state) { - ClusterBlockException blockException = state.blocks().globalBlockedException(ClusterBlockLevel.WRITE); - if (blockException != null) { - if (blockException.retryable()) { - logger.trace("cluster is blocked, scheduling a retry", blockException); - retry(blockException); - } else { - onFailure(blockException); - } - return true; - } - return false; - } - - void retry(Exception failure) { - assert failure != null; - if (observer.isTimedOut()) { - // we running as a last attempt after a timeout has happened. don't retry - onFailure(failure); - return; - } - observer.waitForNextChange(new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - /* - * This is called on the cluster state update thread pool - * but we'd prefer to coordinate the bulk request on the - * write thread pool just to make sure the cluster state - * update thread doesn't get clogged up. - */ - dispatchRetry(); - } - - @Override - public void onClusterServiceClose() { - onFailure(new NodeClosedException(clusterService.localNode())); - } - - @Override - public void onTimeout(TimeValue timeout) { - /* - * Try one more time.... This is called on the generic - * thread pool but out of an abundance of caution we - * switch over to the write thread pool that we expect - * to coordinate the bulk request. - */ - dispatchRetry(); - } - - private void dispatchRetry() { - threadPool.executor(executorName).submit(BulkOperation.this); - } - }); - } - - private boolean addFailureIfRequiresAliasAndAliasIsMissing(DocWriteRequest request, int idx, final Metadata metadata) { - if (request.isRequireAlias() && (metadata.hasAlias(request.index()) == false)) { - Exception exception = new IndexNotFoundException( - "[" + DocWriteRequest.REQUIRE_ALIAS + "] request flag is [true] and [" + request.index() + "] is not an alias", - request.index() - ); - addFailure(request, idx, exception); - return true; - } - return false; - } - - private boolean addFailureIfRequiresDataStreamAndNoParentDataStream(DocWriteRequest request, int idx, final Metadata metadata) { - if (request.isRequireDataStream() && (metadata.indexIsADataStream(request.index()) == false)) { - Exception exception = new ResourceNotFoundException( - "[" - + DocWriteRequest.REQUIRE_DATA_STREAM - + "] request flag is [true] and [" - + request.index() - + "] is not a data stream", - request.index() - ); - addFailure(request, idx, exception); - return true; - } - return false; - } - - private boolean addFailureIfIndexIsClosed(DocWriteRequest request, Index concreteIndex, int idx, final Metadata metadata) { - IndexMetadata indexMetadata = metadata.getIndexSafe(concreteIndex); - if (indexMetadata.getState() == IndexMetadata.State.CLOSE) { - addFailure(request, idx, new IndexClosedException(concreteIndex)); - return true; - } - return false; - } - - private boolean addFailureIfIndexCannotBeCreated(DocWriteRequest request, int idx) { - IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index()); - if (cannotCreate != null) { - addFailure(request, idx, cannotCreate); - return true; - } - return false; - } - - private void addFailure(DocWriteRequest request, int idx, Exception unavailableException) { - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.id(), unavailableException); - BulkItemResponse bulkItemResponse = BulkItemResponse.failure(idx, request.opType(), failure); - responses.set(idx, bulkItemResponse); - // make sure the request gets never processed again - bulkRequest.requests.set(idx, null); - } - } - void executeBulk( Task task, BulkRequest bulkRequest, @@ -906,45 +604,20 @@ void executeBulk( AtomicArray responses, Map indicesThatCannotBeCreated ) { - new BulkOperation(task, bulkRequest, listener, executorName, responses, startTimeNanos, indicesThatCannotBeCreated).run(); - } - - private static class ConcreteIndices { - private final ClusterState state; - private final IndexNameExpressionResolver indexNameExpressionResolver; - private final Map indexAbstractions = new HashMap<>(); - private final Map routings = new HashMap<>(); - - ConcreteIndices(ClusterState state, IndexNameExpressionResolver indexNameExpressionResolver) { - this.state = state; - this.indexNameExpressionResolver = indexNameExpressionResolver; - } - - IndexAbstraction resolveIfAbsent(DocWriteRequest request) { - try { - IndexAbstraction indexAbstraction = indexAbstractions.get(request.index()); - if (indexAbstraction == null) { - indexAbstraction = indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request); - indexAbstractions.put(request.index(), indexAbstraction); - } - return indexAbstraction; - } catch (IndexNotFoundException e) { - if (e.getMetadataKeys().contains(EXCLUDED_DATA_STREAMS_KEY)) { - throw new IllegalArgumentException("only write ops with an op_type of create are allowed in data streams", e); - } else { - throw e; - } - } - } - - IndexRouting routing(Index index) { - IndexRouting routing = routings.get(index); - if (routing == null) { - routing = IndexRouting.fromIndexMetadata(state.metadata().getIndexSafe(index)); - routings.put(index, routing); - } - return routing; - } + new BulkOperation( + task, + threadPool, + executorName, + clusterService, + bulkRequest, + client, + responses, + indicesThatCannotBeCreated, + indexNameExpressionResolver, + relativeTimeProvider, + startTimeNanos, + listener + ).run(); } private long relativeTime() { @@ -955,6 +628,7 @@ private void processBulkIndexIngestRequest( Task task, BulkRequest original, String executorName, + Metadata metadata, ActionListener listener ) { final long ingestStartTimeInNanos = System.nanoTime(); @@ -963,6 +637,8 @@ private void processBulkIndexIngestRequest( original.numberOfActions(), () -> bulkRequestModifier, bulkRequestModifier::markItemAsDropped, + (indexName) -> shouldStoreFailure(indexName, metadata, threadPool.absoluteTimeInMillis()), + bulkRequestModifier::markItemForFailureStore, bulkRequestModifier::markItemAsFailed, (originalThread, exception) -> { if (exception != null) { @@ -1010,137 +686,79 @@ public boolean isForceExecution() { ); } - static final class BulkRequestModifier implements Iterator> { - - final BulkRequest bulkRequest; - final SparseFixedBitSet failedSlots; - final List itemResponses; - final AtomicIntegerArray originalSlots; - - volatile int currentSlot = -1; - - BulkRequestModifier(BulkRequest bulkRequest) { - this.bulkRequest = bulkRequest; - this.failedSlots = new SparseFixedBitSet(bulkRequest.requests().size()); - this.itemResponses = new ArrayList<>(bulkRequest.requests().size()); - this.originalSlots = new AtomicIntegerArray(bulkRequest.requests().size()); // oversize, but that's ok - } - - @Override - public DocWriteRequest next() { - return bulkRequest.requests().get(++currentSlot); - } + /** + * Determines if an index name is associated with either an existing data stream or a template + * for one that has the failure store enabled. + * @param indexName The index name to check. + * @param metadata Cluster state metadata. + * @param epochMillis A timestamp to use when resolving date math in the index name. + * @return true if the given index name corresponds to a data stream with a failure store, + * or if it matches a template that has a data stream failure store enabled. + */ + static boolean shouldStoreFailure(String indexName, Metadata metadata, long epochMillis) { + return DataStream.isFailureStoreEnabled() + && resolveFailureStoreFromMetadata(indexName, metadata, epochMillis).or( + () -> resolveFailureStoreFromTemplate(indexName, metadata) + ).orElse(false); + } - @Override - public boolean hasNext() { - return (currentSlot + 1) < bulkRequest.requests().size(); + /** + * Determines if an index name is associated with an existing data stream that has a failure store enabled. + * @param indexName The index name to check. + * @param metadata Cluster state metadata. + * @param epochMillis A timestamp to use when resolving date math in the index name. + * @return true if the given index name corresponds to an existing data stream with a failure store enabled. + */ + private static Optional resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) { + if (indexName == null) { + return Optional.empty(); } - BulkRequest getBulkRequest() { - if (itemResponses.isEmpty()) { - return bulkRequest; - } else { - BulkRequest modifiedBulkRequest = new BulkRequest(); - modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); - modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); - modifiedBulkRequest.timeout(bulkRequest.timeout()); + // Get index abstraction, resolving date math if it exists + IndexAbstraction indexAbstraction = metadata.getIndicesLookup() + .get(IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis)); - int slot = 0; - List> requests = bulkRequest.requests(); - for (int i = 0; i < requests.size(); i++) { - DocWriteRequest request = requests.get(i); - if (failedSlots.get(i) == false) { - modifiedBulkRequest.add(request); - originalSlots.set(slot++, i); - } - } - return modifiedBulkRequest; - } + // We only store failures if the failure is being written to a data stream, + // not when directly writing to backing indices/failure stores + if (indexAbstraction == null || indexAbstraction.isDataStreamRelated() == false) { + return Optional.empty(); } - ActionListener wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener actionListener) { - if (itemResponses.isEmpty()) { - return actionListener.map( - response -> new BulkResponse(response.getItems(), response.getTook().getMillis(), ingestTookInMillis) - ); - } else { - return actionListener.map(response -> { - // these items are the responses from the subsequent bulk request, their 'slots' - // are not correct for this response we're building - final BulkItemResponse[] bulkResponses = response.getItems(); - - final BulkItemResponse[] allResponses = new BulkItemResponse[bulkResponses.length + itemResponses.size()]; + // Locate the write index for the abstraction, and check if it has a data stream associated with it. + // This handles alias resolution as well as data stream resolution. + Index writeIndex = indexAbstraction.getWriteIndex(); + assert writeIndex != null : "Could not resolve write index for resource [" + indexName + "]"; + IndexAbstraction writeAbstraction = metadata.getIndicesLookup().get(writeIndex.getName()); + DataStream targetDataStream = writeAbstraction.getParentDataStream(); - // the item responses are from the original request, so their slots are correct. - // these are the responses for requests that failed early and were not passed on to the subsequent bulk. - for (BulkItemResponse item : itemResponses) { - allResponses[item.getItemId()] = item; - } - - // use the original slots for the responses from the bulk - for (int i = 0; i < bulkResponses.length; i++) { - allResponses[originalSlots.get(i)] = bulkResponses[i]; - } - - if (Assertions.ENABLED) { - assertResponsesAreCorrect(bulkResponses, allResponses); - } - - return new BulkResponse(allResponses, response.getTook().getMillis(), ingestTookInMillis); - }); - } - } - - private void assertResponsesAreCorrect(BulkItemResponse[] bulkResponses, BulkItemResponse[] allResponses) { - // check for an empty intersection between the ids - final Set failedIds = itemResponses.stream().map(BulkItemResponse::getItemId).collect(Collectors.toSet()); - final Set responseIds = IntStream.range(0, bulkResponses.length) - .map(originalSlots::get) // resolve subsequent bulk ids back to the original slots - .boxed() - .collect(Collectors.toSet()); - assert Sets.haveEmptyIntersection(failedIds, responseIds) - : "bulk item response slots cannot have failed and been processed in the subsequent bulk request, failed ids: " - + failedIds - + ", response ids: " - + responseIds; - - // check for the correct number of responses - final int expectedResponseCount = bulkRequest.requests.size(); - final int actualResponseCount = failedIds.size() + responseIds.size(); - assert expectedResponseCount == actualResponseCount - : "Expected [" + expectedResponseCount + "] responses, but found [" + actualResponseCount + "]"; + // We will store the failure if the write target belongs to a data stream with a failure store. + return Optional.of(targetDataStream != null && targetDataStream.isFailureStore()); + } - // check that every response is present - for (int i = 0; i < allResponses.length; i++) { - assert allResponses[i] != null : "BulkItemResponse at index [" + i + "] was null"; + /** + * Determines if an index name is associated with an index template that has a data stream failure store enabled. + * @param indexName The index name to check. + * @param metadata Cluster state metadata. + * @return true if the given index name corresponds to an index template with a data stream failure store enabled. + */ + private static Optional resolveFailureStoreFromTemplate(String indexName, Metadata metadata) { + if (indexName == null) { + return Optional.empty(); + } + + // Check to see if the index name matches any templates such that an index would have been attributed + // We don't check v1 templates at all because failure stores can only exist on data streams via a v2 template + String template = MetadataIndexTemplateService.findV2Template(metadata, indexName, false); + if (template != null) { + // Check if this is a data stream template or if it is just a normal index. + ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template); + if (composableIndexTemplate.getDataStreamTemplate() != null) { + // Check if the data stream has the failure store enabled + return Optional.of(composableIndexTemplate.getDataStreamTemplate().hasFailureStore()); } } - synchronized void markItemAsFailed(int slot, Exception e) { - final DocWriteRequest docWriteRequest = bulkRequest.requests().get(slot); - final String id = Objects.requireNonNullElse(docWriteRequest.id(), DROPPED_OR_FAILED_ITEM_WITH_AUTO_GENERATED_ID); - // We hit a error during preprocessing a request, so we: - // 1) Remember the request item slot from the bulk, so that when we're done processing all requests we know what failed - // 2) Add a bulk item failure for this request - // 3) Continue with the next request in the bulk. - failedSlots.set(slot); - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(docWriteRequest.index(), id, e); - itemResponses.add(BulkItemResponse.failure(slot, docWriteRequest.opType(), failure)); - } - - synchronized void markItemAsDropped(int slot) { - final DocWriteRequest docWriteRequest = bulkRequest.requests().get(slot); - final String id = Objects.requireNonNullElse(docWriteRequest.id(), DROPPED_OR_FAILED_ITEM_WITH_AUTO_GENERATED_ID); - failedSlots.set(slot); - UpdateResponse dropped = new UpdateResponse( - new ShardId(docWriteRequest.index(), IndexMetadata.INDEX_UUID_NA_VALUE, 0), - id, - UNASSIGNED_SEQ_NO, - UNASSIGNED_PRIMARY_TERM, - docWriteRequest.version(), - DocWriteResponse.Result.NOOP - ); - itemResponses.add(BulkItemResponse.success(slot, docWriteRequest.opType(), dropped)); - } + // Could not locate a failure store via template + return Optional.empty(); } } diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java index f99bea1a64821..f2b1dc7cd556c 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.delete; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; @@ -21,15 +22,24 @@ public class DeleteRequestBuilder extends ReplicationRequestBuilder { + private String id; + private String routing; + private Long version; + private VersionType versionType; + private Long seqNo; + private Long term; + private WriteRequest.RefreshPolicy refreshPolicy; + public DeleteRequestBuilder(ElasticsearchClient client, @Nullable String index) { - super(client, TransportDeleteAction.TYPE, new DeleteRequest(index)); + super(client, TransportDeleteAction.TYPE); + setIndex(index); } /** * Sets the id of the document to delete. */ public DeleteRequestBuilder setId(String id) { - request.id(id); + this.id = id; return this; } @@ -38,7 +48,7 @@ public DeleteRequestBuilder setId(String id) { * and not the id. */ public DeleteRequestBuilder setRouting(String routing) { - request.routing(routing); + this.routing = routing; return this; } @@ -47,7 +57,7 @@ public DeleteRequestBuilder setRouting(String routing) { * version exists and no changes happened on the doc since then. */ public DeleteRequestBuilder setVersion(long version) { - request.version(version); + this.version = version; return this; } @@ -55,7 +65,7 @@ public DeleteRequestBuilder setVersion(long version) { * Sets the type of versioning to use. Defaults to {@link VersionType#INTERNAL}. */ public DeleteRequestBuilder setVersionType(VersionType versionType) { - request.versionType(versionType); + this.versionType = versionType; return this; } @@ -67,7 +77,7 @@ public DeleteRequestBuilder setVersionType(VersionType versionType) { * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. */ public DeleteRequestBuilder setIfSeqNo(long seqNo) { - request.setIfSeqNo(seqNo); + this.seqNo = seqNo; return this; } @@ -79,8 +89,47 @@ public DeleteRequestBuilder setIfSeqNo(long seqNo) { * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. */ public DeleteRequestBuilder setIfPrimaryTerm(long term) { - request.setIfPrimaryTerm(term); + this.term = term; + return this; + } + + @Override + public DeleteRequestBuilder setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + this.refreshPolicy = refreshPolicy; return this; } + @Override + public DeleteRequestBuilder setRefreshPolicy(String refreshPolicy) { + this.refreshPolicy = WriteRequest.RefreshPolicy.parse(refreshPolicy); + return this; + } + + @Override + public DeleteRequest request() { + DeleteRequest request = new DeleteRequest(); + super.apply(request); + if (id != null) { + request.id(id); + } + if (routing != null) { + request.routing(routing); + } + if (version != null) { + request.version(version); + } + if (versionType != null) { + request.versionType(versionType); + } + if (seqNo != null) { + request.setIfSeqNo(seqNo); + } + if (term != null) { + request.setIfPrimaryTerm(term); + } + if (refreshPolicy != null) { + request.setRefreshPolicy(refreshPolicy); + } + return request; + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index 969d86f5f470c..641ca33d8e05b 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -57,8 +57,15 @@ FieldCapabilitiesIndexResponse fetch( ) throws IOException { final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); final IndexShard indexShard = indexService.getShard(shardId.getId()); - // no need to open a searcher if we aren't filtering - try (Engine.Searcher searcher = alwaysMatches(indexFilter) ? null : indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) { + final Engine.Searcher searcher; + if (alwaysMatches(indexFilter)) { + // no need to open a searcher if we aren't filtering, but make sure we are reading from an up-to-dated shard + indexShard.readAllowed(); + searcher = null; + } else { + searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE); + } + try (searcher) { return doFetch( task, shardId, diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 5eab04663e959..d26545fd8acca 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.NoShardAvailableActionException; -import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.indices.refresh.TransportShardRefreshAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.BasicReplicationRequest; @@ -194,6 +193,10 @@ private void handleGetOnUnpromotableShard(GetRequest request, IndexShard indexSh ShardId shardId = indexShard.shardId(); if (request.refresh()) { var node = getCurrentNodeOfPrimary(clusterService.state(), shardId); + if (node == null) { + listener.onFailure(new NoShardAvailableActionException(shardId, "primary shard is not active")); + return; + } logger.trace("send refresh action for shard {} to node {}", shardId, node.getId()); var refreshRequest = new BasicReplicationRequest(shardId); refreshRequest.setParentTask(request.getParentTask()); @@ -230,10 +233,7 @@ private void getFromTranslog( tryGetFromTranslog(request, indexShard, state, listener.delegateResponse((l, e) -> { final var cause = ExceptionsHelper.unwrapCause(e); logger.debug("get_from_translog failed", cause); - if (cause instanceof ShardNotFoundException - || cause instanceof IndexNotFoundException - || cause instanceof NoShardAvailableActionException - || cause instanceof UnavailableShardsException) { + if (cause instanceof ShardNotFoundException || cause instanceof IndexNotFoundException) { logger.debug("retrying get_from_translog"); observer.waitForNextChange(new ClusterStateObserver.Listener() { @Override @@ -260,6 +260,10 @@ public void onTimeout(TimeValue timeout) { private void tryGetFromTranslog(GetRequest request, IndexShard indexShard, ClusterState state, ActionListener listener) { ShardId shardId = indexShard.shardId(); var node = getCurrentNodeOfPrimary(state, shardId); + if (node == null) { + listener.onFailure(new NoShardAvailableActionException(shardId, "primary shard is not active")); + return; + } TransportGetFromTranslogAction.Request getFromTranslogRequest = new TransportGetFromTranslogAction.Request(request, shardId); getFromTranslogRequest.setParentTask(request.getParentTask()); transportService.sendRequest( @@ -296,7 +300,7 @@ private void tryGetFromTranslog(GetRequest request, IndexShard indexShard, Clust static DiscoveryNode getCurrentNodeOfPrimary(ClusterState clusterState, ShardId shardId) { var shardRoutingTable = clusterState.routingTable().shardRoutingTable(shardId); if (shardRoutingTable.primaryShard() == null || shardRoutingTable.primaryShard().active() == false) { - throw new NoShardAvailableActionException(shardId, "primary shard is not active"); + return null; } DiscoveryNode node = clusterState.nodes().get(shardRoutingTable.primaryShard().currentNodeId()); assert node != null; diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 6dfd706b3268f..2e558b42d7e2b 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -8,10 +8,15 @@ package org.elasticsearch.action.get; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.admin.indices.refresh.TransportShardRefreshAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportActions; @@ -19,6 +24,7 @@ import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.OperationRouting; @@ -27,15 +33,17 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardNotFoundException; import org.elasticsearch.indices.ExecutorSelector; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.logging.LogManager; -import org.elasticsearch.logging.Logger; +import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -163,8 +171,12 @@ private void handleMultiGetOnUnpromotableShard( ActionListener listener ) throws IOException { ShardId shardId = indexShard.shardId(); - var node = getCurrentNodeOfPrimary(clusterService.state(), shardId); if (request.refresh()) { + var node = getCurrentNodeOfPrimary(clusterService.state(), shardId); + if (node == null) { + listener.onFailure(new NoShardAvailableActionException(shardId, "primary shard is not active")); + return; + } logger.trace("send refresh action for shard {} to node {}", shardId, node.getId()); var refreshRequest = new BasicReplicationRequest(shardId); refreshRequest.setParentTask(request.getParentTask()); @@ -173,57 +185,116 @@ private void handleMultiGetOnUnpromotableShard( refreshRequest, listener.delegateFailureAndWrap((l, replicationResponse) -> super.asyncShardOperation(request, shardId, l)) ); - } else if (request.realtime()) { - TransportShardMultiGetFomTranslogAction.Request mgetFromTranslogRequest = new TransportShardMultiGetFomTranslogAction.Request( - request, - shardId - ); - mgetFromTranslogRequest.setParentTask(request.getParentTask()); - transportService.sendRequest( - node, - TransportShardMultiGetFomTranslogAction.NAME, - mgetFromTranslogRequest, - new ActionListenerResponseHandler<>(listener.delegateFailure((l, r) -> { - var responseHasMissingLocations = false; - for (int i = 0; i < r.multiGetShardResponse().locations.size(); i++) { - if (r.multiGetShardResponse().responses.get(i) == null && r.multiGetShardResponse().failures.get(i) == null) { - responseHasMissingLocations = true; - break; - } - } - if (responseHasMissingLocations == false) { - logger.debug("received result of all ids in real-time mget[shard] from the promotable shard."); - l.onResponse(r.multiGetShardResponse()); - } else { - logger.debug( - "no result for some ids from the promotable shard (segment generation to wait for: {})", - r.segmentGeneration() - ); - if (r.segmentGeneration() == -1) { - // Nothing to wait for (no previous unsafe generation), just handle the rest locally. - ActionRunnable.supply(l, () -> handleLocalGets(request, r.multiGetShardResponse(), shardId)).run(); - } else { - assert r.segmentGeneration() > -1L; - assert r.primaryTerm() > Engine.UNKNOWN_PRIMARY_TERM; - indexShard.waitForPrimaryTermAndGeneration( - r.primaryTerm(), - r.segmentGeneration(), - listener.delegateFailureAndWrap( - (ll, aLong) -> getExecutor(request, shardId).execute( - ActionRunnable.supply(ll, () -> handleLocalGets(request, r.multiGetShardResponse(), shardId)) - ) - ) - ); - } - } - }), TransportShardMultiGetFomTranslogAction.Response::new, getExecutor(request, shardId)) + return; + } + if (request.realtime()) { + final var state = clusterService.state(); + final var observer = new ClusterStateObserver( + state, + clusterService, + TimeValue.timeValueSeconds(60), + logger, + threadPool.getThreadContext() ); + shardMultiGetFromTranslog(request, indexShard, state, observer, listener); } else { // A non-real-time mget with no explicit refresh requested. super.asyncShardOperation(request, shardId, listener); } } + private void shardMultiGetFromTranslog( + MultiGetShardRequest request, + IndexShard indexShard, + ClusterState state, + ClusterStateObserver observer, + ActionListener listener + ) { + tryShardMultiGetFromTranslog(request, indexShard, state, listener.delegateResponse((l, e) -> { + final var cause = ExceptionsHelper.unwrapCause(e); + logger.debug("mget_from_translog[shard] failed", cause); + if (cause instanceof ShardNotFoundException || cause instanceof IndexNotFoundException) { + logger.debug("retrying mget_from_translog[shard]"); + observer.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + shardMultiGetFromTranslog(request, indexShard, state, observer, l); + } + + @Override + public void onClusterServiceClose() { + l.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + l.onFailure(new ElasticsearchException("Timed out retrying mget_from_translog[shard]", cause)); + } + }); + } else { + l.onFailure(e); + } + })); + } + + private void tryShardMultiGetFromTranslog( + MultiGetShardRequest request, + IndexShard indexShard, + ClusterState state, + ActionListener listener + ) { + final var shardId = indexShard.shardId(); + var node = getCurrentNodeOfPrimary(state, shardId); + if (node == null) { + listener.onFailure(new NoShardAvailableActionException(shardId, "primary shard is not active")); + return; + } + TransportShardMultiGetFomTranslogAction.Request mgetFromTranslogRequest = new TransportShardMultiGetFomTranslogAction.Request( + request, + shardId + ); + mgetFromTranslogRequest.setParentTask(request.getParentTask()); + transportService.sendRequest( + node, + TransportShardMultiGetFomTranslogAction.NAME, + mgetFromTranslogRequest, + new ActionListenerResponseHandler<>(listener.delegateFailure((l, r) -> { + var responseHasMissingLocations = false; + for (int i = 0; i < r.multiGetShardResponse().locations.size(); i++) { + if (r.multiGetShardResponse().responses.get(i) == null && r.multiGetShardResponse().failures.get(i) == null) { + responseHasMissingLocations = true; + break; + } + } + if (responseHasMissingLocations == false) { + logger.debug("received result of all ids in real-time mget[shard] from the promotable shard."); + l.onResponse(r.multiGetShardResponse()); + } else { + logger.debug( + "no result for some ids from the promotable shard (segment generation to wait for: {})", + r.segmentGeneration() + ); + if (r.segmentGeneration() == -1) { + // Nothing to wait for (no previous unsafe generation), just handle the rest locally. + ActionRunnable.supply(l, () -> handleLocalGets(request, r.multiGetShardResponse(), shardId)).run(); + } else { + assert r.segmentGeneration() > -1L; + assert r.primaryTerm() > Engine.UNKNOWN_PRIMARY_TERM; + indexShard.waitForPrimaryTermAndGeneration( + r.primaryTerm(), + r.segmentGeneration(), + listener.delegateFailureAndWrap( + (ll, aLong) -> getExecutor(request, shardId).execute( + ActionRunnable.supply(ll, () -> handleLocalGets(request, r.multiGetShardResponse(), shardId)) + ) + ) + ); + } + } + }), TransportShardMultiGetFomTranslogAction.Response::new, getExecutor(request, shardId)) + ); + } + private MultiGetShardResponse handleLocalGets(MultiGetShardRequest request, MultiGetShardResponse response, ShardId shardId) { logger.trace("handling local gets for missing locations"); for (int i = 0; i < response.locations.size(); i++) { diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java index 5058990efd966..52504176eb7e1 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.util.Objects; -// TODO(ES-5727): add a retry mechanism to TransportShardMultiGetFromTranslogAction public class TransportShardMultiGetFomTranslogAction extends HandledTransportAction< TransportShardMultiGetFomTranslogAction.Request, TransportShardMultiGetFomTranslogAction.Response> { diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index b1ad328abda92..bf2384f56576d 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -19,6 +20,7 @@ import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.client.internal.Requests; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRouting; @@ -110,6 +112,11 @@ public class IndexRequest extends ReplicatedWriteRequest implement private boolean requireDataStream; + /** + * Transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire. + */ + private boolean writeToFailureStore = false; + /** * This indicates whether the response to this request ought to list the ingest pipelines that were executed on the document */ @@ -477,6 +484,18 @@ public IndexRequest source(Object... source) { *

*/ public IndexRequest source(XContentType xContentType, Object... source) { + return source(getXContentBuilder(xContentType, source)); + } + + /** + * Returns an XContentBuilder for the given xContentType and source array + *

+ * Note: the number of objects passed to this method as varargs must be an even + * number. Also the first argument in each pair (the field name) must have a + * valid String representation. + *

+ */ + public static XContentBuilder getXContentBuilder(XContentType xContentType, Object... source) { if (source.length % 2 != 0) { throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]"); } @@ -489,11 +508,14 @@ public IndexRequest source(XContentType xContentType, Object... source) { try { XContentBuilder builder = XContentFactory.contentBuilder(xContentType); builder.startObject(); - for (int i = 0; i < source.length; i++) { - builder.field(source[i++].toString(), source[i]); + // This for loop increments by 2 because the source array contains adjacent key/value pairs: + for (int i = 0; i < source.length; i = i + 2) { + String field = source[i].toString(); + Object value = source[i + 1]; + builder.field(field, value); } builder.endObject(); - return source(builder); + return builder; } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate", e); } @@ -821,7 +843,25 @@ public IndexRequest setRequireDataStream(boolean requireDataStream) { @Override public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { - return ia.getWriteIndex(this, metadata); + if (DataStream.isFailureStoreEnabled() && writeToFailureStore) { + if (ia.isDataStreamRelated() == false) { + throw new ElasticsearchException( + "Attempting to write a document to a failure store but the targeted index is not a data stream" + ); + } + // Resolve write index and get parent data stream to handle the case of dealing with an alias + String defaultWriteIndexName = ia.getWriteIndex().getName(); + DataStream dataStream = metadata.getIndicesLookup().get(defaultWriteIndexName).getParentDataStream(); + if (dataStream.getFailureIndices().size() < 1) { + throw new ElasticsearchException( + "Attempting to write a document to a failure store but the target data stream does not have one enabled" + ); + } + return dataStream.getFailureIndices().get(dataStream.getFailureIndices().size() - 1); + } else { + // Resolve as normal + return ia.getWriteIndex(this, metadata); + } } @Override @@ -834,6 +874,15 @@ public IndexRequest setRequireAlias(boolean requireAlias) { return this; } + public boolean isWriteToFailureStore() { + return writeToFailureStore; + } + + public IndexRequest setWriteToFailureStore(boolean writeToFailureStore) { + this.writeToFailureStore = writeToFailureStore; + return this; + } + public IndexRequest setListExecutedPipelines(boolean listExecutedPipelines) { this.listExecutedPipelines = listExecutedPipelines; return this; diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index b8faf39514cbe..0cb04fbdba1a6 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -8,17 +8,23 @@ package org.elasticsearch.action.index; +import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; +import org.elasticsearch.client.internal.Requests; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.VersionType; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import java.io.IOException; import java.util.Map; /** @@ -27,13 +33,30 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder implements WriteRequestBuilder { + private String id = null; + + private BytesReference sourceBytesReference; + private XContentType sourceContentType; + + private String pipeline; + private Boolean requireAlias; + private Boolean requireDataStream; + private String routing; + private WriteRequest.RefreshPolicy refreshPolicy; + private Long ifSeqNo; + private Long ifPrimaryTerm; + private DocWriteRequest.OpType opType; + private Boolean create; + private Long version; + private VersionType versionType; public IndexRequestBuilder(ElasticsearchClient client) { - super(client, TransportIndexAction.TYPE, new IndexRequest()); + this(client, null); } public IndexRequestBuilder(ElasticsearchClient client, @Nullable String index) { - super(client, TransportIndexAction.TYPE, new IndexRequest(index)); + super(client, TransportIndexAction.TYPE); + setIndex(index); } /** @@ -41,7 +64,7 @@ public IndexRequestBuilder(ElasticsearchClient client, @Nullable String index) { * generated. */ public IndexRequestBuilder setId(String id) { - request.id(id); + this.id = id; return this; } @@ -50,7 +73,7 @@ public IndexRequestBuilder setId(String id) { * and not the id. */ public IndexRequestBuilder setRouting(String routing) { - request.routing(routing); + this.routing = routing; return this; } @@ -58,7 +81,8 @@ public IndexRequestBuilder setRouting(String routing) { * Sets the source. */ public IndexRequestBuilder setSource(BytesReference source, XContentType xContentType) { - request.source(source, xContentType); + this.sourceBytesReference = source; + this.sourceContentType = xContentType; return this; } @@ -68,8 +92,7 @@ public IndexRequestBuilder setSource(BytesReference source, XContentType xConten * @param source The map to index */ public IndexRequestBuilder setSource(Map source) { - request.source(source); - return this; + return setSource(source, Requests.INDEX_CONTENT_TYPE); } /** @@ -78,8 +101,13 @@ public IndexRequestBuilder setSource(Map source) { * @param source The map to index */ public IndexRequestBuilder setSource(Map source, XContentType contentType) { - request.source(source, contentType); - return this; + try { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + builder.map(source); + return setSource(builder); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate", e); + } } /** @@ -89,7 +117,8 @@ public IndexRequestBuilder setSource(Map source, XContentType content * or using the {@link #setSource(byte[], XContentType)}. */ public IndexRequestBuilder setSource(String source, XContentType xContentType) { - request.source(source, xContentType); + this.sourceBytesReference = new BytesArray(source); + this.sourceContentType = xContentType; return this; } @@ -97,7 +126,8 @@ public IndexRequestBuilder setSource(String source, XContentType xContentType) { * Sets the content source to index. */ public IndexRequestBuilder setSource(XContentBuilder sourceBuilder) { - request.source(sourceBuilder); + this.sourceBytesReference = BytesReference.bytes(sourceBuilder); + this.sourceContentType = sourceBuilder.contentType(); return this; } @@ -105,8 +135,7 @@ public IndexRequestBuilder setSource(XContentBuilder sourceBuilder) { * Sets the document to index in bytes form. */ public IndexRequestBuilder setSource(byte[] source, XContentType xContentType) { - request.source(source, xContentType); - return this; + return setSource(source, 0, source.length, xContentType); } /** @@ -119,7 +148,8 @@ public IndexRequestBuilder setSource(byte[] source, XContentType xContentType) { * @param xContentType The type/format of the source */ public IndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) { - request.source(source, offset, length, xContentType); + this.sourceBytesReference = new BytesArray(source, offset, length); + this.sourceContentType = xContentType; return this; } @@ -132,8 +162,7 @@ public IndexRequestBuilder setSource(byte[] source, int offset, int length, XCon *

*/ public IndexRequestBuilder setSource(Object... source) { - request.source(source); - return this; + return setSource(Requests.INDEX_CONTENT_TYPE, source); } /** @@ -145,15 +174,14 @@ public IndexRequestBuilder setSource(Object... source) { *

*/ public IndexRequestBuilder setSource(XContentType xContentType, Object... source) { - request.source(xContentType, source); - return this; + return setSource(IndexRequest.getXContentBuilder(xContentType, source)); } /** * Sets the type of operation to perform. */ public IndexRequestBuilder setOpType(DocWriteRequest.OpType opType) { - request.opType(opType); + this.opType = opType; return this; } @@ -161,7 +189,7 @@ public IndexRequestBuilder setOpType(DocWriteRequest.OpType opType) { * Set to {@code true} to force this index to use {@link org.elasticsearch.action.index.IndexRequest.OpType#CREATE}. */ public IndexRequestBuilder setCreate(boolean create) { - request.create(create); + this.create = create; return this; } @@ -170,7 +198,7 @@ public IndexRequestBuilder setCreate(boolean create) { * version exists and no changes happened on the doc since then. */ public IndexRequestBuilder setVersion(long version) { - request.version(version); + this.version = version; return this; } @@ -178,7 +206,7 @@ public IndexRequestBuilder setVersion(long version) { * Sets the versioning type. Defaults to {@link VersionType#INTERNAL}. */ public IndexRequestBuilder setVersionType(VersionType versionType) { - request.versionType(versionType); + this.versionType = versionType; return this; } @@ -190,7 +218,7 @@ public IndexRequestBuilder setVersionType(VersionType versionType) { * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. */ public IndexRequestBuilder setIfSeqNo(long seqNo) { - request.setIfSeqNo(seqNo); + this.ifSeqNo = seqNo; return this; } @@ -202,7 +230,7 @@ public IndexRequestBuilder setIfSeqNo(long seqNo) { * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. */ public IndexRequestBuilder setIfPrimaryTerm(long term) { - request.setIfPrimaryTerm(term); + this.ifPrimaryTerm = term; return this; } @@ -210,7 +238,7 @@ public IndexRequestBuilder setIfPrimaryTerm(long term) { * Sets the ingest pipeline to be executed before indexing the document */ public IndexRequestBuilder setPipeline(String pipeline) { - request.setPipeline(pipeline); + this.pipeline = pipeline; return this; } @@ -218,7 +246,7 @@ public IndexRequestBuilder setPipeline(String pipeline) { * Sets the require_alias flag */ public IndexRequestBuilder setRequireAlias(boolean requireAlias) { - request.setRequireAlias(requireAlias); + this.requireAlias = requireAlias; return this; } @@ -226,7 +254,64 @@ public IndexRequestBuilder setRequireAlias(boolean requireAlias) { * Sets the require_data_stream flag */ public IndexRequestBuilder setRequireDataStream(boolean requireDataStream) { - request.setRequireDataStream(requireDataStream); + this.requireDataStream = requireDataStream; return this; } + + public IndexRequestBuilder setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + this.refreshPolicy = refreshPolicy; + return this; + } + + public IndexRequestBuilder setRefreshPolicy(String refreshPolicy) { + this.refreshPolicy = WriteRequest.RefreshPolicy.parse(refreshPolicy); + return this; + } + + @Override + public IndexRequest request() { + IndexRequest request = new IndexRequest(); + super.apply(request); + request.id(id); + if (sourceBytesReference != null && sourceContentType != null) { + request.source(sourceBytesReference, sourceContentType); + } + if (pipeline != null) { + request.setPipeline(pipeline); + } + if (routing != null) { + request.routing(routing); + } + if (refreshPolicy != null) { + request.setRefreshPolicy(refreshPolicy); + } + if (ifSeqNo != null) { + request.setIfSeqNo(ifSeqNo); + } + if (ifPrimaryTerm != null) { + request.setIfPrimaryTerm(ifPrimaryTerm); + } + if (pipeline != null) { + request.setPipeline(pipeline); + } + if (requireAlias != null) { + request.setRequireAlias(requireAlias); + } + if (requireDataStream != null) { + request.setRequireDataStream(requireDataStream); + } + if (opType != null) { + request.opType(opType); + } + if (create != null) { + request.create(create); + } + if (version != null) { + request.version(version); + } + if (versionType != null) { + request.versionType(versionType); + } + return request; + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 6cfea93068a86..cde9bd91e421f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -62,6 +62,8 @@ import java.util.function.Consumer; import java.util.function.Supplier; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; + public final class SearchPhaseController { private static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0]; @@ -137,10 +139,10 @@ public static List mergeKnnResults(SearchRequest request, List> topDocsLists = new ArrayList<>(request.source().knnSearch().size()); - List> nestedPath = new ArrayList<>(request.source().knnSearch().size()); - for (int i = 0; i < request.source().knnSearch().size(); i++) { + SearchSourceBuilder source = request.source(); + List> topDocsLists = new ArrayList<>(source.knnSearch().size()); + List> nestedPath = new ArrayList<>(source.knnSearch().size()); + for (int i = 0; i < source.knnSearch().size(); i++) { topDocsLists.add(new ArrayList<>()); nestedPath.add(new SetOnce<>()); } @@ -159,9 +161,9 @@ public static List mergeKnnResults(SearchRequest request, List mergedResults = new ArrayList<>(request.source().knnSearch().size()); - for (int i = 0; i < request.source().knnSearch().size(); i++) { - TopDocs mergedTopDocs = TopDocs.merge(request.source().knnSearch().get(i).k(), topDocsLists.get(i).toArray(new TopDocs[0])); + List mergedResults = new ArrayList<>(source.knnSearch().size()); + for (int i = 0; i < source.knnSearch().size(); i++) { + TopDocs mergedTopDocs = TopDocs.merge(source.knnSearch().get(i).k(), topDocsLists.get(i).toArray(new TopDocs[0])); mergedResults.add(new DfsKnnResults(nestedPath.get(i).get(), mergedTopDocs.scoreDocs)); } return mergedResults; @@ -706,12 +708,10 @@ private static void validateMergeSortValueFormats(Collection buildPerIndexOriginalIndices( @@ -284,7 +288,8 @@ public long buildTookInMillis() { @Override protected void doExecute(Task task, SearchRequest searchRequest, ActionListener listener) { - ActionListener loggingListener = listener.delegateFailureAndWrap((l, searchResponse) -> { + ActionListener loggingAndMetrics = listener.delegateFailureAndWrap((l, searchResponse) -> { + searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { // Deduplicate failures by exception message and index ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures()); @@ -301,7 +306,7 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener< } l.onResponse(searchResponse); }); - executeRequest((SearchTask) task, searchRequest, loggingListener, AsyncSearchActionProvider::new); + executeRequest((SearchTask) task, searchRequest, loggingAndMetrics, AsyncSearchActionProvider::new); } void executeRequest( diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java index 27086366283f9..c1c43310b0e11 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.rest.action.search.SearchResponseMetrics; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; @@ -36,22 +37,26 @@ public class TransportSearchScrollAction extends HandledTransportAction listener) { - ActionListener loggingListener = listener.delegateFailureAndWrap((l, searchResponse) -> { + ActionListener loggingAndMetrics = listener.delegateFailureAndWrap((l, searchResponse) -> { + searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures()); for (ShardOperationFailedException f : groupedFailures) { @@ -74,7 +79,7 @@ protected void doExecute(Task task, SearchScrollRequest request, ActionListener< request, (SearchTask) task, scrollId, - loggingListener + loggingAndMetrics ); case QUERY_AND_FETCH_TYPE -> // TODO can we get rid of this? new SearchScrollQueryAndFetchAsyncAction( @@ -84,7 +89,7 @@ protected void doExecute(Task task, SearchScrollRequest request, ActionListener< request, (SearchTask) task, scrollId, - loggingListener + loggingAndMetrics ); default -> throw new IllegalArgumentException("Scroll id type [" + scrollId.getType() + "] unrecognized"); }; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java index a4d5e07103df3..8eb82af2091cd 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java @@ -8,7 +8,7 @@ package org.elasticsearch.action.support.replication; -import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestLazyBuilder; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActiveShardCount; @@ -18,18 +18,23 @@ public abstract class ReplicationRequestBuilder< Request extends ReplicationRequest, Response extends ActionResponse, - RequestBuilder extends ReplicationRequestBuilder> extends ActionRequestBuilder { + RequestBuilder extends ReplicationRequestBuilder> extends ActionRequestLazyBuilder< + Request, + Response> { + private String index; + private TimeValue timeout; + private ActiveShardCount waitForActiveShards; - protected ReplicationRequestBuilder(ElasticsearchClient client, ActionType action, Request request) { - super(client, action, request); + protected ReplicationRequestBuilder(ElasticsearchClient client, ActionType action) { + super(client, action); } /** * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}. */ @SuppressWarnings("unchecked") - public final RequestBuilder setTimeout(TimeValue timeout) { - request.timeout(timeout); + public RequestBuilder setTimeout(TimeValue timeout) { + this.timeout = timeout; return (RequestBuilder) this; } @@ -37,24 +42,28 @@ public final RequestBuilder setTimeout(TimeValue timeout) { * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}. */ @SuppressWarnings("unchecked") - public final RequestBuilder setTimeout(String timeout) { - request.timeout(timeout); + public RequestBuilder setTimeout(String timeout) { + this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"); return (RequestBuilder) this; } @SuppressWarnings("unchecked") - public final RequestBuilder setIndex(String index) { - request.index(index); + public RequestBuilder setIndex(String index) { + this.index = index; return (RequestBuilder) this; } + public String getIndex() { + return index; + } + /** * Sets the number of shard copies that must be active before proceeding with the write. * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details. */ @SuppressWarnings("unchecked") public RequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { - request.waitForActiveShards(waitForActiveShards); + this.waitForActiveShards = waitForActiveShards; return (RequestBuilder) this; } @@ -66,4 +75,17 @@ public RequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShard public RequestBuilder setWaitForActiveShards(final int waitForActiveShards) { return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } + + protected void apply(Request request) { + if (index != null) { + request.index(index); + } + if (timeout != null) { + request.timeout(timeout); + } + if (waitForActiveShards != null) { + request.waitForActiveShards(waitForActiveShards); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java index 931f072e1e45e..64efcda2f14db 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java @@ -8,7 +8,7 @@ package org.elasticsearch.action.support.single.instance; -import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestLazyBuilder; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.client.internal.ElasticsearchClient; @@ -17,26 +17,32 @@ public abstract class InstanceShardOperationRequestBuilder< Request extends InstanceShardOperationRequest, Response extends ActionResponse, - RequestBuilder extends InstanceShardOperationRequestBuilder> extends ActionRequestBuilder< + RequestBuilder extends InstanceShardOperationRequestBuilder> extends ActionRequestLazyBuilder< Request, Response> { + private String index; + private TimeValue timeout; - protected InstanceShardOperationRequestBuilder(ElasticsearchClient client, ActionType action, Request request) { - super(client, action, request); + protected InstanceShardOperationRequestBuilder(ElasticsearchClient client, ActionType action) { + super(client, action); } @SuppressWarnings("unchecked") - public final RequestBuilder setIndex(String index) { - request.index(index); + public RequestBuilder setIndex(String index) { + this.index = index; return (RequestBuilder) this; } + protected String getIndex() { + return index; + } + /** * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}. */ @SuppressWarnings("unchecked") - public final RequestBuilder setTimeout(TimeValue timeout) { - request.timeout(timeout); + public RequestBuilder setTimeout(TimeValue timeout) { + this.timeout = timeout; return (RequestBuilder) this; } @@ -44,8 +50,17 @@ public final RequestBuilder setTimeout(TimeValue timeout) { * A timeout to wait if the index operation can't be performed immediately. Defaults to {@code 1m}. */ @SuppressWarnings("unchecked") - public final RequestBuilder setTimeout(String timeout) { - request.timeout(timeout); + public RequestBuilder setTimeout(String timeout) { + this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"); return (RequestBuilder) this; } + + protected void apply(Request request) { + if (index != null) { + request.index(index); + } + if (timeout != null) { + request.timeout(timeout); + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index d7b1ea46b77b0..36b6cc6aa9964 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -704,6 +704,14 @@ private IndexRequest safeDoc() { return doc; } + /** + * Sets the doc source of the update request to be used when the document does not exists. + */ + public UpdateRequest upsert(BytesReference source, XContentType contentType) { + safeUpsertRequest().source(source, contentType); + return this; + } + /** * Sets the index request to be used if the document does not exists. Otherwise, a * {@link org.elasticsearch.index.engine.DocumentMissingException} is thrown. diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 88bed844558f2..c1ee0f7b8af37 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -8,37 +8,77 @@ package org.elasticsearch.action.update; +import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; +import org.elasticsearch.client.internal.Requests; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Script; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import java.io.IOException; import java.util.Map; public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder implements WriteRequestBuilder { + private String id; + private String routing; + private Script script; + + private String fetchSourceInclude; + private String fetchSourceExclude; + private String[] fetchSourceIncludeArray; + private String[] fetchSourceExcludeArray; + private Boolean fetchSource; + + private Integer retryOnConflict; + private Long version; + private VersionType versionType; + private Long ifSeqNo; + private Long ifPrimaryTerm; + private ActiveShardCount waitForActiveShards; + + private IndexRequest doc; + private BytesReference docSourceBytesReference; + private XContentType docSourceXContentType; + + private IndexRequest upsert; + private BytesReference upsertSourceBytesReference; + private XContentType upsertSourceXContentType; + + private Boolean docAsUpsert; + private Boolean detectNoop; + private Boolean scriptedUpsert; + private Boolean requireAlias; + private WriteRequest.RefreshPolicy refreshPolicy; + public UpdateRequestBuilder(ElasticsearchClient client) { - super(client, TransportUpdateAction.TYPE, new UpdateRequest()); + this(client, null, null); } public UpdateRequestBuilder(ElasticsearchClient client, String index, String id) { - super(client, TransportUpdateAction.TYPE, new UpdateRequest(index, id)); + super(client, TransportUpdateAction.TYPE); + setIndex(index); + setId(id); } /** * Sets the id of the indexed document. */ public UpdateRequestBuilder setId(String id) { - request.id(id); + this.id = id; return this; } @@ -47,7 +87,7 @@ public UpdateRequestBuilder setId(String id) { * and not the id. */ public UpdateRequestBuilder setRouting(String routing) { - request.routing(routing); + this.routing = routing; return this; } @@ -60,7 +100,7 @@ public UpdateRequestBuilder setRouting(String routing) { * */ public UpdateRequestBuilder setScript(Script script) { - request.script(script); + this.script = script; return this; } @@ -77,7 +117,8 @@ public UpdateRequestBuilder setScript(Script script) { * the returned _source */ public UpdateRequestBuilder setFetchSource(@Nullable String include, @Nullable String exclude) { - request.fetchSource(include, exclude); + this.fetchSourceInclude = include; + this.fetchSourceExclude = exclude; return this; } @@ -94,7 +135,8 @@ public UpdateRequestBuilder setFetchSource(@Nullable String include, @Nullable S * filter the returned _source */ public UpdateRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) { - request.fetchSource(includes, excludes); + this.fetchSourceIncludeArray = includes; + this.fetchSourceExcludeArray = excludes; return this; } @@ -102,7 +144,7 @@ public UpdateRequestBuilder setFetchSource(@Nullable String[] includes, @Nullabl * Indicates whether the response should contain the updated _source. */ public UpdateRequestBuilder setFetchSource(boolean fetchSource) { - request.fetchSource(fetchSource); + this.fetchSource = fetchSource; return this; } @@ -111,7 +153,7 @@ public UpdateRequestBuilder setFetchSource(boolean fetchSource) { * getting it and updating it. Defaults to 0. */ public UpdateRequestBuilder setRetryOnConflict(int retryOnConflict) { - request.retryOnConflict(retryOnConflict); + this.retryOnConflict = retryOnConflict; return this; } @@ -120,7 +162,7 @@ public UpdateRequestBuilder setRetryOnConflict(int retryOnConflict) { * version exists and no changes happened on the doc since then. */ public UpdateRequestBuilder setVersion(long version) { - request.version(version); + this.version = version; return this; } @@ -128,7 +170,7 @@ public UpdateRequestBuilder setVersion(long version) { * Sets the versioning type. Defaults to {@link org.elasticsearch.index.VersionType#INTERNAL}. */ public UpdateRequestBuilder setVersionType(VersionType versionType) { - request.versionType(versionType); + this.versionType = versionType; return this; } @@ -140,7 +182,7 @@ public UpdateRequestBuilder setVersionType(VersionType versionType) { * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. */ public UpdateRequestBuilder setIfSeqNo(long seqNo) { - request.setIfSeqNo(seqNo); + this.ifSeqNo = seqNo; return this; } @@ -152,7 +194,7 @@ public UpdateRequestBuilder setIfSeqNo(long seqNo) { * {@link org.elasticsearch.index.engine.VersionConflictEngineException} will be thrown. */ public UpdateRequestBuilder setIfPrimaryTerm(long term) { - request.setIfPrimaryTerm(term); + this.ifPrimaryTerm = term; return this; } @@ -161,7 +203,7 @@ public UpdateRequestBuilder setIfPrimaryTerm(long term) { * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details. */ public UpdateRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { - request.waitForActiveShards(waitForActiveShards); + this.waitForActiveShards = waitForActiveShards; return this; } @@ -178,7 +220,7 @@ public UpdateRequestBuilder setWaitForActiveShards(final int waitForActiveShards * Sets the doc to use for updates when a script is not specified. */ public UpdateRequestBuilder setDoc(IndexRequest indexRequest) { - request.doc(indexRequest); + this.doc = indexRequest; return this; } @@ -186,7 +228,8 @@ public UpdateRequestBuilder setDoc(IndexRequest indexRequest) { * Sets the doc to use for updates when a script is not specified. */ public UpdateRequestBuilder setDoc(XContentBuilder source) { - request.doc(source); + this.docSourceBytesReference = BytesReference.bytes(source); + this.docSourceXContentType = source.contentType(); return this; } @@ -194,23 +237,28 @@ public UpdateRequestBuilder setDoc(XContentBuilder source) { * Sets the doc to use for updates when a script is not specified. */ public UpdateRequestBuilder setDoc(Map source) { - request.doc(source); - return this; + return setDoc(source, Requests.INDEX_CONTENT_TYPE); } /** * Sets the doc to use for updates when a script is not specified. */ public UpdateRequestBuilder setDoc(Map source, XContentType contentType) { - request.doc(source, contentType); - return this; + try { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + builder.map(source); + return setDoc(builder); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } } /** * Sets the doc to use for updates when a script is not specified. */ public UpdateRequestBuilder setDoc(String source, XContentType xContentType) { - request.doc(source, xContentType); + this.docSourceBytesReference = new BytesArray(source); + this.docSourceXContentType = xContentType; return this; } @@ -218,15 +266,15 @@ public UpdateRequestBuilder setDoc(String source, XContentType xContentType) { * Sets the doc to use for updates when a script is not specified. */ public UpdateRequestBuilder setDoc(byte[] source, XContentType xContentType) { - request.doc(source, xContentType); - return this; + return setDoc(source, 0, source.length, xContentType); } /** * Sets the doc to use for updates when a script is not specified. */ public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XContentType xContentType) { - request.doc(source, offset, length, xContentType); + this.docSourceBytesReference = new BytesArray(source, offset, length); + this.docSourceXContentType = xContentType; return this; } @@ -235,8 +283,7 @@ public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XConte * is a field and value pairs. */ public UpdateRequestBuilder setDoc(Object... source) { - request.doc(source); - return this; + return setDoc(Requests.INDEX_CONTENT_TYPE, source); } /** @@ -244,8 +291,7 @@ public UpdateRequestBuilder setDoc(Object... source) { * is a field and value pairs. */ public UpdateRequestBuilder setDoc(XContentType xContentType, Object... source) { - request.doc(xContentType, source); - return this; + return setDoc(IndexRequest.getXContentBuilder(xContentType, source)); } /** @@ -253,7 +299,7 @@ public UpdateRequestBuilder setDoc(XContentType xContentType, Object... source) * {@link org.elasticsearch.index.engine.DocumentMissingException} is thrown. */ public UpdateRequestBuilder setUpsert(IndexRequest indexRequest) { - request.upsert(indexRequest); + this.upsert = indexRequest; return this; } @@ -261,7 +307,8 @@ public UpdateRequestBuilder setUpsert(IndexRequest indexRequest) { * Sets the doc source of the update request to be used when the document does not exists. */ public UpdateRequestBuilder setUpsert(XContentBuilder source) { - request.upsert(source); + this.upsertSourceBytesReference = BytesReference.bytes(source); + this.upsertSourceXContentType = source.contentType(); return this; } @@ -269,23 +316,28 @@ public UpdateRequestBuilder setUpsert(XContentBuilder source) { * Sets the doc source of the update request to be used when the document does not exists. */ public UpdateRequestBuilder setUpsert(Map source) { - request.upsert(source); - return this; + return setUpsert(source, Requests.INDEX_CONTENT_TYPE); } /** * Sets the doc source of the update request to be used when the document does not exists. */ public UpdateRequestBuilder setUpsert(Map source, XContentType contentType) { - request.upsert(source, contentType); - return this; + try { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + builder.map(source); + return setUpsert(builder); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } } /** * Sets the doc source of the update request to be used when the document does not exists. */ public UpdateRequestBuilder setUpsert(String source, XContentType xContentType) { - request.upsert(source, xContentType); + this.upsertSourceBytesReference = new BytesArray(source); + this.upsertSourceXContentType = xContentType; return this; } @@ -293,15 +345,15 @@ public UpdateRequestBuilder setUpsert(String source, XContentType xContentType) * Sets the doc source of the update request to be used when the document does not exists. */ public UpdateRequestBuilder setUpsert(byte[] source, XContentType xContentType) { - request.upsert(source, xContentType); - return this; + return setUpsert(source, 0, source.length, xContentType); } /** * Sets the doc source of the update request to be used when the document does not exists. */ public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XContentType xContentType) { - request.upsert(source, offset, length, xContentType); + this.upsertSourceBytesReference = new BytesArray(source, offset, length); + this.upsertSourceXContentType = xContentType; return this; } @@ -310,8 +362,7 @@ public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XCo * includes field and value pairs. */ public UpdateRequestBuilder setUpsert(Object... source) { - request.upsert(source); - return this; + return setUpsert(Requests.INDEX_CONTENT_TYPE, source); } /** @@ -319,15 +370,14 @@ public UpdateRequestBuilder setUpsert(Object... source) { * includes field and value pairs. */ public UpdateRequestBuilder setUpsert(XContentType xContentType, Object... source) { - request.upsert(xContentType, source); - return this; + return setUpsert(IndexRequest.getXContentBuilder(xContentType, source)); } /** * Sets whether the specified doc parameter should be used as upsert document. */ public UpdateRequestBuilder setDocAsUpsert(boolean shouldUpsertDoc) { - request.docAsUpsert(shouldUpsertDoc); + this.docAsUpsert = shouldUpsertDoc; return this; } @@ -336,7 +386,7 @@ public UpdateRequestBuilder setDocAsUpsert(boolean shouldUpsertDoc) { * Defaults to true. */ public UpdateRequestBuilder setDetectNoop(boolean detectNoop) { - request.detectNoop(detectNoop); + this.detectNoop = detectNoop; return this; } @@ -344,7 +394,7 @@ public UpdateRequestBuilder setDetectNoop(boolean detectNoop) { * Sets whether the script should be run in the case of an insert */ public UpdateRequestBuilder setScriptedUpsert(boolean scriptedUpsert) { - request.scriptedUpsert(scriptedUpsert); + this.scriptedUpsert = scriptedUpsert; return this; } @@ -352,7 +402,127 @@ public UpdateRequestBuilder setScriptedUpsert(boolean scriptedUpsert) { * Sets the require_alias flag */ public UpdateRequestBuilder setRequireAlias(boolean requireAlias) { - request.setRequireAlias(requireAlias); - return this; + this.requireAlias = requireAlias; + return this; + } + + @Override + public UpdateRequestBuilder setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + this.refreshPolicy = refreshPolicy; + return this; + } + + @Override + public UpdateRequestBuilder setRefreshPolicy(String refreshPolicy) { + this.refreshPolicy = WriteRequest.RefreshPolicy.parse(refreshPolicy); + return this; + } + + @Override + public UpdateRequest request() { + validate(); + UpdateRequest request = new UpdateRequest(); + super.apply(request); + if (id != null) { + request.id(id); + } + if (routing != null) { + request.routing(routing); + } + if (script != null) { + request.script(script); + } + if (fetchSourceInclude != null || fetchSourceExclude != null) { + request.fetchSource(fetchSourceInclude, fetchSourceExclude); + } + if (fetchSourceIncludeArray != null || fetchSourceExcludeArray != null) { + request.fetchSource(fetchSourceIncludeArray, fetchSourceExcludeArray); + } + if (fetchSource != null) { + request.fetchSource(fetchSource); + } + if (retryOnConflict != null) { + request.retryOnConflict(retryOnConflict); + } + if (version != null) { + request.version(version); + } + if (versionType != null) { + request.versionType(versionType); + } + if (ifSeqNo != null) { + request.setIfSeqNo(ifSeqNo); + } + if (ifPrimaryTerm != null) { + request.setIfPrimaryTerm(ifPrimaryTerm); + } + if (waitForActiveShards != null) { + request.waitForActiveShards(waitForActiveShards); + } + if (doc != null) { + request.doc(doc); + } + if (docSourceBytesReference != null && docSourceXContentType != null) { + request.doc(docSourceBytesReference, docSourceXContentType); + } + if (upsert != null) { + request.upsert(upsert); + } + if (upsertSourceBytesReference != null && upsertSourceXContentType != null) { + request.upsert(upsertSourceBytesReference, upsertSourceXContentType); + } + if (docAsUpsert != null) { + request.docAsUpsert(docAsUpsert); + } + if (detectNoop != null) { + request.detectNoop(detectNoop); + } + if (scriptedUpsert != null) { + request.scriptedUpsert(scriptedUpsert); + } + if (requireAlias != null) { + request.setRequireAlias(requireAlias); + } + if (refreshPolicy != null) { + request.setRefreshPolicy(refreshPolicy); + } + return request; + } + + protected void validate() throws IllegalStateException { + boolean fetchIncludeExcludeNotNull = fetchSourceInclude != null || fetchSourceExclude != null; + boolean fetchIncludeExcludeArrayNotNull = fetchSourceIncludeArray != null || fetchSourceExcludeArray != null; + boolean fetchSourceNotNull = fetchSource != null; + if ((fetchIncludeExcludeNotNull && fetchIncludeExcludeArrayNotNull) + || (fetchIncludeExcludeNotNull && fetchSourceNotNull) + || (fetchIncludeExcludeArrayNotNull && fetchSourceNotNull)) { + throw new IllegalStateException("Only one fetchSource() method may be called"); + } + int docSourceFieldsSet = countDocSourceFieldsSet(); + if (docSourceFieldsSet > 1) { + throw new IllegalStateException("Only one setDoc() method may be called, but " + docSourceFieldsSet + " have been"); + } + int upsertSourceFieldsSet = countUpsertSourceFieldsSet(); + if (upsertSourceFieldsSet > 1) { + throw new IllegalStateException("Only one setUpsert() method may be called, but " + upsertSourceFieldsSet + " have been"); + } + } + + private int countDocSourceFieldsSet() { + return countNonNullObjects(doc, docSourceBytesReference); + } + + private int countUpsertSourceFieldsSet() { + return countNonNullObjects(upsert, upsertSourceBytesReference); + } + + private int countNonNullObjects(Object... objects) { + int sum = 0; + for (Object object : objects) { + if (object != null) { + sum++; + } + } + return sum; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java index 6307ed768e813..7e2c0849a6fad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java @@ -151,7 +151,8 @@ private static ClusterState applyRequest( MapperService.mergeMappings( mapperService.documentMapper(), mapping, - request.autoUpdate() ? MergeReason.MAPPING_AUTO_UPDATE : MergeReason.MAPPING_UPDATE + request.autoUpdate() ? MergeReason.MAPPING_AUTO_UPDATE : MergeReason.MAPPING_UPDATE, + mapperService.getIndexSettings() ); } Metadata.Builder builder = Metadata.builder(metadata); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 0d40bd2d08c14..1ed9d759c4ca8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -248,6 +248,10 @@ public static class ExtractFromSource extends IndexRouting { this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); } + public boolean matchesField(String fieldName) { + return isRoutingPath.test(fieldName); + } + @Override public void process(IndexRequest indexRequest) {} diff --git a/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java b/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java index 2fca882724bbd..1e30579292d00 100644 --- a/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java @@ -143,6 +143,14 @@ public int read() throws IOException { @Override public int read(final byte[] b, final int bOffset, final int len) throws IOException { + if (slice.remaining() >= len) { + slice.get(b, bOffset, len); + return len; + } + return readFromMultipleSlices(b, bOffset, len); + } + + private int readFromMultipleSlices(byte[] b, int bOffset, int len) throws IOException { final int length = bytesReference.length(); final int offset = offset(); if (offset >= length) { @@ -186,6 +194,14 @@ public long skip(long n) throws IOException { if (n <= 0L) { return 0L; } + if (n <= slice.remaining()) { + slice.position(slice.position() + (int) n); + return n; + } + return skipMultiple(n); + } + + private int skipMultiple(long n) throws IOException { assert offset() <= bytesReference.length() : offset() + " vs " + bytesReference.length(); // definitely >= 0 and <= Integer.MAX_VALUE so casting is ok final int numBytesSkipped = (int) Math.min(n, bytesReference.length() - offset()); diff --git a/server/src/main/java/org/elasticsearch/common/bytes/ReleasableBytesReference.java b/server/src/main/java/org/elasticsearch/common/bytes/ReleasableBytesReference.java index 567f39d968200..e9fe63529e17a 100644 --- a/server/src/main/java/org/elasticsearch/common/bytes/ReleasableBytesReference.java +++ b/server/src/main/java/org/elasticsearch/common/bytes/ReleasableBytesReference.java @@ -144,7 +144,7 @@ public long ramBytesUsed() { @Override public StreamInput streamInput() throws IOException { assert hasReferences(); - return new BytesReferenceStreamInput(this) { + return new BytesReferenceStreamInput(delegate) { private ReleasableBytesReference retainAndSkip(int len) throws IOException { if (len == 0) { return ReleasableBytesReference.empty(); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index c4f0dc58f5ffd..9e271ee6f9bfc 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -364,7 +364,7 @@ public Text readOptionalText() throws IOException { } public Text readText() throws IOException { - // use StringAndBytes so we can cache the string if its ever converted to it + // use StringAndBytes so we can cache the string if it's ever converted to it int length = readInt(); return new Text(readBytesReference(length)); } @@ -1271,8 +1271,8 @@ protected int readArraySize() throws IOException { if (arraySize < 0) { throwNegative(arraySize); } - // lets do a sanity check that if we are reading an array size that is bigger that the remaining bytes we can safely - // throw an exception instead of allocating the array based on the size. A simple corrutpted byte can make a node go OOM + // let's do a sanity check that if we are reading an array size that is bigger that the remaining bytes we can safely + // throw an exception instead of allocating the array based on the size. A simple corrupted byte can make a node go OOM // if the size is large and for perf reasons we allocate arrays ahead of time ensureCanReadBytes(arraySize); return arraySize; @@ -1287,7 +1287,7 @@ private static void throwExceedsMaxArraySize(int arraySize) { } /** - * This method throws an {@link EOFException} if the given number of bytes can not be read from the this stream. This method might + * This method throws an {@link EOFException} if the given number of bytes can not be read from the stream. This method might * be a no-op depending on the underlying implementation if the information of the remaining bytes is not present. */ protected abstract void ensureCanReadBytes(int length) throws EOFException; diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index a0b62bdabc08b..a3350c4526a91 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -596,10 +596,13 @@ public final void writeMap(final Map< * @param valueWriter The value writer */ public final void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { - writeVInt(map.size()); - for (final Map.Entry entry : map.entrySet()) { - keyWriter.write(this, entry.getKey()); - valueWriter.write(this, entry.getValue()); + int size = map.size(); + writeVInt(size); + if (size > 0) { + for (final Map.Entry entry : map.entrySet()) { + keyWriter.write(this, entry.getKey()); + valueWriter.write(this, entry.getValue()); + } } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index c1b8d51c255db..41dd840b0c0e7 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -151,6 +151,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING, MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionHandler.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionHandler.java index 9457773eb8071..f8e1eabe0440b 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionHandler.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionHandler.java @@ -34,8 +34,8 @@ protected void incrementRejections() { } } - public void registerCounter(MeterRegistry meterRegistry, String prefix, String name) { - rejectionCounter = meterRegistry.registerLongCounter(prefix + ".rejected.total", "number of rejected threads for " + name, "count"); + public void registerCounter(MeterRegistry meterRegistry, String metric_name, String threadpool_name) { + rejectionCounter = meterRegistry.registerLongCounter(metric_name, "number of rejected threads for " + threadpool_name, "count"); rejectionCounter.incrementBy(rejected()); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 2e600bbdc5ed4..c5a5e5a5c4b96 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -651,6 +651,18 @@ public SearchExecutionContext newSearchExecutionContext( LongSupplier nowInMillis, String clusterAlias, Map runtimeMappings + ) { + return newSearchExecutionContext(shardId, shardRequestIndex, searcher, nowInMillis, clusterAlias, runtimeMappings, null); + } + + public SearchExecutionContext newSearchExecutionContext( + int shardId, + int shardRequestIndex, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Map runtimeMappings, + Integer requestSize ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher( index().getName(), @@ -677,7 +689,8 @@ public SearchExecutionContext newSearchExecutionContext( indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, - runtimeMappings + runtimeMappings, + requestSize ); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 83a6d9319c75a..6decd20e0a41f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -42,6 +42,7 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING; +import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; @@ -753,6 +754,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingNestedFieldsLimit; private volatile long mappingNestedDocsLimit; private volatile long mappingTotalFieldsLimit; + private volatile boolean ignoreDynamicFieldsBeyondLimit; private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; private volatile long mappingDimensionFieldsLimit; @@ -897,6 +899,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingNestedFieldsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); + ignoreDynamicFieldsBeyondLimit = scopedSettings.get(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING); mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING); mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); @@ -976,6 +979,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING, this::setRetentionLeaseMillis); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, this::setMappingNestedFieldsLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, this::setMappingNestedDocsLimit); + scopedSettings.addSettingsUpdateConsumer( + INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, + this::setIgnoreDynamicFieldsBeyondLimit + ); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); @@ -1519,6 +1526,14 @@ private void setMappingTotalFieldsLimit(long value) { this.mappingTotalFieldsLimit = value; } + private void setIgnoreDynamicFieldsBeyondLimit(boolean ignoreDynamicFieldsBeyondLimit) { + this.ignoreDynamicFieldsBeyondLimit = ignoreDynamicFieldsBeyondLimit; + } + + public boolean isIgnoreDynamicFieldsBeyondLimit() { + return ignoreDynamicFieldsBeyondLimit; + } + public long getMappingDepthLimit() { return mappingDepthLimit; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index ce321b012ab95..ae6185cdcc6b6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -100,6 +100,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_8_12_1_LUCENE_9_9_2 = def(8_500_010, Version.LUCENE_9_9_2); public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_00_0, Version.LUCENE_9_9_1); public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); + public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java new file mode 100644 index 0000000000000..1813601fc9477 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.FlatVectorsFormat; +import org.apache.lucene.codecs.FlatVectorsReader; +import org.apache.lucene.codecs.FlatVectorsWriter; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; + +import java.io.IOException; + +public class ES813FlatVectorFormat extends KnnVectorsFormat { + + static final String NAME = "ES813FlatVectorFormat"; + + private final FlatVectorsFormat format = new Lucene99FlatVectorsFormat(); + + /** + * Sole constructor + */ + public ES813FlatVectorFormat() { + super(NAME); + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new ES813FlatVectorWriter(format.fieldsWriter(state)); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES813FlatVectorReader(format.fieldsReader(state)); + } + + public static class ES813FlatVectorWriter extends KnnVectorsWriter { + + private final FlatVectorsWriter writer; + + public ES813FlatVectorWriter(FlatVectorsWriter writer) { + super(); + this.writer = writer; + } + + @Override + public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + return writer.addField(fieldInfo, null); + } + + @Override + public void flush(int maxDoc, Sorter.DocMap sortMap) throws IOException { + writer.flush(maxDoc, sortMap); + } + + @Override + public void finish() throws IOException { + writer.finish(); + } + + @Override + public void close() throws IOException { + writer.close(); + } + + @Override + public long ramBytesUsed() { + return writer.ramBytesUsed(); + } + + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + writer.mergeOneField(fieldInfo, mergeState); + } + } + + public static class ES813FlatVectorReader extends KnnVectorsReader { + + private final FlatVectorsReader reader; + + public ES813FlatVectorReader(FlatVectorsReader reader) { + super(); + this.reader = reader; + } + + @Override + public void checkIntegrity() throws IOException { + reader.checkIntegrity(); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + return reader.getFloatVectorValues(field); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return reader.getByteVectorValues(field); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + private void collectAllMatchingDocs(KnnCollector knnCollector, Bits acceptDocs, RandomVectorScorer scorer) throws IOException { + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + assert collector.earlyTerminated() == false; + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + @Override + public void close() throws IOException { + reader.close(); + } + + @Override + public long ramBytesUsed() { + return reader.ramBytesUsed(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java new file mode 100644 index 0000000000000..5764f31d018c4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.FlatVectorsFormat; +import org.apache.lucene.codecs.FlatVectorsReader; +import org.apache.lucene.codecs.FlatVectorsWriter; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; + +import java.io.IOException; + +public class ES813Int8FlatVectorFormat extends KnnVectorsFormat { + + static final String NAME = "ES813Int8FlatVectorFormat"; + + private final FlatVectorsFormat format; + + public ES813Int8FlatVectorFormat() { + this(null); + } + + /** + * Sole constructor + */ + public ES813Int8FlatVectorFormat(Float confidenceInterval) { + super(NAME); + this.format = new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval); + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new ES813FlatVectorWriter(format.fieldsWriter(state)); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES813FlatVectorReader(format.fieldsReader(state)); + } + + public static class ES813FlatVectorWriter extends KnnVectorsWriter { + + private final FlatVectorsWriter writer; + + public ES813FlatVectorWriter(FlatVectorsWriter writer) { + super(); + this.writer = writer; + } + + @Override + public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + return writer.addField(fieldInfo, null); + } + + @Override + public void flush(int maxDoc, Sorter.DocMap sortMap) throws IOException { + writer.flush(maxDoc, sortMap); + } + + @Override + public void finish() throws IOException { + writer.finish(); + } + + @Override + public void close() throws IOException { + writer.close(); + } + + @Override + public long ramBytesUsed() { + return writer.ramBytesUsed(); + } + + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + writer.mergeOneField(fieldInfo, mergeState); + } + } + + public static class ES813FlatVectorReader extends KnnVectorsReader { + + private final FlatVectorsReader reader; + + public ES813FlatVectorReader(FlatVectorsReader reader) { + super(); + this.reader = reader; + } + + @Override + public void checkIntegrity() throws IOException { + reader.checkIntegrity(); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + return reader.getFloatVectorValues(field); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return reader.getByteVectorValues(field); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + private void collectAllMatchingDocs(KnnCollector knnCollector, Bits acceptDocs, RandomVectorScorer scorer) throws IOException { + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + assert collector.earlyTerminated() == false; + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + @Override + public void close() throws IOException { + reader.close(); + } + + @Override + public long ramBytesUsed() { + return reader.ramBytesUsed(); + } + + } +} diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 3849095a94e6e..a910e496ce1b5 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -629,11 +629,6 @@ public DeleteResult(Exception failure, long version, long term, long seqNo, bool this.found = found; } - public DeleteResult(Mapping requiredMappingUpdate, String id) { - super(Operation.TYPE.DELETE, requiredMappingUpdate, id); - this.found = false; - } - public boolean isFound() { return found; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java index fafa7f7a9cb12..3cf90f2385525 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java @@ -32,17 +32,19 @@ static DocumentDimensions fromIndexSettings(IndexSettings indexSettings) { * value is already computed in some cases when we want to collect * dimensions, so we can save re-computing the UTF-8 encoding. */ - void addString(String fieldName, BytesRef utf8Value); + DocumentDimensions addString(String fieldName, BytesRef utf8Value); - default void addString(String fieldName, String value) { - addString(fieldName, new BytesRef(value)); + default DocumentDimensions addString(String fieldName, String value) { + return addString(fieldName, new BytesRef(value)); } - void addIp(String fieldName, InetAddress value); + DocumentDimensions addIp(String fieldName, InetAddress value); - void addLong(String fieldName, long value); + DocumentDimensions addLong(String fieldName, long value); - void addUnsignedLong(String fieldName, long value); + DocumentDimensions addUnsignedLong(String fieldName, long value); + + DocumentDimensions validate(IndexSettings settings); /** * Makes sure that each dimension only appears on time. @@ -51,29 +53,40 @@ class OnlySingleValueAllowed implements DocumentDimensions { private final Set names = new HashSet<>(); @Override - public void addString(String fieldName, BytesRef value) { + public DocumentDimensions addString(String fieldName, BytesRef value) { add(fieldName); + return this; } // Override to skip the UTF-8 conversion that happens in the default implementation @Override - public void addString(String fieldName, String value) { + public DocumentDimensions addString(String fieldName, String value) { add(fieldName); + return this; } @Override - public void addIp(String fieldName, InetAddress value) { + public DocumentDimensions addIp(String fieldName, InetAddress value) { add(fieldName); + return this; } @Override - public void addLong(String fieldName, long value) { + public DocumentDimensions addLong(String fieldName, long value) { add(fieldName); + return this; } @Override - public void addUnsignedLong(String fieldName, long value) { + public DocumentDimensions addUnsignedLong(String fieldName, long value) { add(fieldName); + return this; + } + + @Override + public DocumentDimensions validate(final IndexSettings settings) { + // DO NOTHING + return this; } private void add(String fieldName) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 54223e1e692f3..fe6b0b2051dc9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -514,7 +514,11 @@ private static void parseObjectDynamic(DocumentParserContext context, String cur } if (context.dynamic() != ObjectMapper.Dynamic.RUNTIME) { - context.addDynamicMapper(dynamicObjectMapper); + if (context.addDynamicMapper(dynamicObjectMapper) == false) { + failIfMatchesRoutingPath(context, currentFieldName); + context.parser().skipChildren(); + return; + } } if (dynamicObjectMapper instanceof NestedObjectMapper && context.isWithinCopyTo()) { throwOnCreateDynamicNestedViaCopyTo(dynamicObjectMapper, context); @@ -556,7 +560,10 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr parseNonDynamicArray(context, currentFieldName, currentFieldName); } else { if (parsesArrayValue(objectMapperFromTemplate)) { - context.addDynamicMapper(objectMapperFromTemplate); + if (context.addDynamicMapper(objectMapperFromTemplate) == false) { + context.parser().skipChildren(); + return; + } context.path().add(currentFieldName); parseObjectOrField(context, objectMapperFromTemplate); context.path().remove(); @@ -674,7 +681,9 @@ private static void parseDynamicValue(final DocumentParserContext context, Strin failIfMatchesRoutingPath(context, currentFieldName); return; } - context.dynamic().getDynamicFieldsBuilder().createDynamicFieldFromValue(context, currentFieldName); + if (context.dynamic().getDynamicFieldsBuilder().createDynamicFieldFromValue(context, currentFieldName) == false) { + failIfMatchesRoutingPath(context, currentFieldName); + } } private static void ensureNotStrict(DocumentParserContext context, String currentFieldName) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index b9dfc83d17683..0a669fb0ade8a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -80,20 +80,38 @@ protected void addDoc(LuceneDocument doc) { } } + /** + * Tracks the number of dynamically added mappers. + * All {@link DocumentParserContext}s that are created via {@link DocumentParserContext#createChildContext(ObjectMapper)} + * share the same mutable instance so that we can track the total size of dynamic mappers + * that are added on any level of the object graph. + */ + private static final class DynamicMapperSize { + private int dynamicMapperSize = 0; + + public void add(int mapperSize) { + dynamicMapperSize += mapperSize; + } + + public int get() { + return dynamicMapperSize; + } + } + private final MappingLookup mappingLookup; private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; private final Set ignoredFields; private final Map> dynamicMappers; - private final Set newFieldsSeen; + private final DynamicMapperSize dynamicMappersSize; private final Map dynamicObjectMappers; - private final List dynamicRuntimeFields; + private final Map> dynamicRuntimeFields; private final DocumentDimensions dimensions; private final ObjectMapper parent; private final ObjectMapper.Dynamic dynamic; private String id; private Field version; - private SeqNoFieldMapper.SequenceIDFields seqID; + private final SeqNoFieldMapper.SequenceIDFields seqID; private final Set fieldsAppliedFromTemplates; private final Set copyToFields; @@ -103,9 +121,8 @@ private DocumentParserContext( SourceToParse sourceToParse, Set ignoreFields, Map> dynamicMappers, - Set newFieldsSeen, Map dynamicObjectMappers, - List dynamicRuntimeFields, + Map> dynamicRuntimeFields, String id, Field version, SeqNoFieldMapper.SequenceIDFields seqID, @@ -113,14 +130,14 @@ private DocumentParserContext( ObjectMapper parent, ObjectMapper.Dynamic dynamic, Set fieldsAppliedFromTemplates, - Set copyToFields + Set copyToFields, + DynamicMapperSize dynamicMapperSize ) { this.mappingLookup = mappingLookup; this.mappingParserContext = mappingParserContext; this.sourceToParse = sourceToParse; this.ignoredFields = ignoreFields; this.dynamicMappers = dynamicMappers; - this.newFieldsSeen = newFieldsSeen; this.dynamicObjectMappers = dynamicObjectMappers; this.dynamicRuntimeFields = dynamicRuntimeFields; this.id = id; @@ -131,6 +148,7 @@ private DocumentParserContext( this.dynamic = dynamic; this.fieldsAppliedFromTemplates = fieldsAppliedFromTemplates; this.copyToFields = copyToFields; + this.dynamicMappersSize = dynamicMapperSize; } private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, DocumentParserContext in) { @@ -140,7 +158,6 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.sourceToParse, in.ignoredFields, in.dynamicMappers, - in.newFieldsSeen, in.dynamicObjectMappers, in.dynamicRuntimeFields, in.id, @@ -150,7 +167,8 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, parent, dynamic, in.fieldsAppliedFromTemplates, - in.copyToFields + in.copyToFields, + in.dynamicMappersSize ); } @@ -167,17 +185,17 @@ protected DocumentParserContext( source, new HashSet<>(), new HashMap<>(), - new HashSet<>(), new HashMap<>(), - new ArrayList<>(), - null, + new HashMap<>(), null, null, + SeqNoFieldMapper.SequenceIDFields.emptySeqID(), DocumentDimensions.fromIndexSettings(mappingParserContext.getIndexSettings()), parent, dynamic, new HashSet<>(), - new HashSet<>() + new HashSet<>(), + new DynamicMapperSize() ); } @@ -264,10 +282,6 @@ public final SeqNoFieldMapper.SequenceIDFields seqID() { return this.seqID; } - public final void seqID(SeqNoFieldMapper.SequenceIDFields seqID) { - this.seqID = seqID; - } - /** * Description on the document being parsed used in error messages. Not * called unless there is an error. @@ -303,8 +317,13 @@ public boolean isCopyToField(String name) { /** * Add a new mapper dynamically created while parsing. + * + * @return returns true if the mapper could be created, false if the dynamic mapper has been ignored due to + * the field limit + * @throws IllegalArgumentException if the field limit has been exceeded. + * This can happen when dynamic is set to {@link ObjectMapper.Dynamic#TRUE} or {@link ObjectMapper.Dynamic#RUNTIME}. */ - public final void addDynamicMapper(Mapper mapper) { + public final boolean addDynamicMapper(Mapper mapper) { // eagerly check object depth limit here to avoid stack overflow errors if (mapper instanceof ObjectMapper) { MappingLookup.checkObjectDepthLimit(indexSettings().getMappingDepthLimit(), mapper.name()); @@ -315,8 +334,18 @@ public final void addDynamicMapper(Mapper mapper) { // note that existing fields can also receive dynamic mapping updates (e.g. constant_keyword to fix the value) if (mappingLookup.getMapper(mapper.name()) == null && mappingLookup.objectMappers().containsKey(mapper.name()) == false - && newFieldsSeen.add(mapper.name())) { - mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), newFieldsSeen.size()); + && dynamicMappers.containsKey(mapper.name()) == false) { + int mapperSize = mapper.mapperSize(); + int additionalFieldsToAdd = getNewFieldsSize() + mapperSize; + if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) { + if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd)) { + addIgnoredField(mapper.name()); + return false; + } + } else { + mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd); + } + dynamicMappersSize.add(mapperSize); } if (mapper instanceof ObjectMapper objectMapper) { dynamicObjectMappers.put(objectMapper.name(), objectMapper); @@ -337,6 +366,25 @@ public final void addDynamicMapper(Mapper mapper) { // 1) by default, they would be empty containers in the mappings, is it then important to map them? // 2) they can be the result of applying a dynamic template which may define sub-fields or set dynamic, enabled or subobjects. dynamicMappers.computeIfAbsent(mapper.name(), k -> new ArrayList<>()).add(mapper); + return true; + } + + /* + * Returns an approximation of the number of dynamically mapped fields and runtime fields that will be added to the mapping. + * This is to validate early and to fail fast during document parsing. + * There will be another authoritative (but more expensive) validation step when making the actual update mapping request. + * During the mapping update, the actual number fields is determined by counting the total number of fields of the merged mapping. + * Therefore, both over-counting and under-counting here is not critical. + * However, in order for users to get to the field limit, we should try to be as close as possible to the actual field count. + * If we under-count fields here, we may only know that we exceed the field limit during the mapping update. + * This can happen when merging the mappers for the same field results in a mapper with a larger size than the individual mappers. + * This leads to document rejection instead of ignoring fields above the limit + * if ignore_dynamic_beyond_limit is configured for the index. + * If we over-count the fields (for example by counting all mappers with the same name), + * we may reject fields earlier than necessary and before actually hitting the field limit. + */ + int getNewFieldsSize() { + return dynamicMappersSize.get() + dynamicRuntimeFields.size(); } /** @@ -395,11 +443,19 @@ final ObjectMapper getDynamicObjectMapper(String name) { * because for dynamic mappings, a new field can be either mapped * as runtime or indexed, but never both. */ - final void addDynamicRuntimeField(RuntimeField runtimeField) { - if (newFieldsSeen.add(runtimeField.name())) { - mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), newFieldsSeen.size()); + final boolean addDynamicRuntimeField(RuntimeField runtimeField) { + if (dynamicRuntimeFields.containsKey(runtimeField.name()) == false) { + if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) { + if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), getNewFieldsSize() + 1)) { + addIgnoredField(runtimeField.name()); + return false; + } + } else { + mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), getNewFieldsSize() + 1); + } } - dynamicRuntimeFields.add(runtimeField); + dynamicRuntimeFields.computeIfAbsent(runtimeField.name(), k -> new ArrayList<>(1)).add(runtimeField); + return true; } /** @@ -408,7 +464,7 @@ final void addDynamicRuntimeField(RuntimeField runtimeField) { * or when dynamic templates specify a runtime section. */ public final List getDynamicRuntimeFields() { - return Collections.unmodifiableList(dynamicRuntimeFields); + return dynamicRuntimeFields.values().stream().flatMap(List::stream).toList(); } /** @@ -552,7 +608,12 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() { if (p.endsWith(".")) { p = p.substring(0, p.length() - 1); } - return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false); + boolean containsDimensions = false; + ObjectMapper objectMapper = mappingLookup.objectMappers().get(p); + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + containsDimensions = passThroughObjectMapper.containsDimensions(); + } + return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false, containsDimensions); } public abstract XContentParser parser(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index f2d1b8058f115..8505c561bfb1a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -10,9 +10,9 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.XContentParser; @@ -44,7 +44,7 @@ private DynamicFieldsBuilder(Strategy strategy) { * delegates to the appropriate strategy which depends on the current dynamic mode. * The strategy defines if fields are going to be mapped as ordinary or runtime fields. */ - void createDynamicFieldFromValue(final DocumentParserContext context, String name) throws IOException { + boolean createDynamicFieldFromValue(final DocumentParserContext context, String name) throws IOException { XContentParser.Token token = context.parser().currentToken(); if (token == XContentParser.Token.VALUE_STRING) { String text = context.parser().text(); @@ -66,14 +66,14 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam } if (parseableAsLong && context.root().numericDetection()) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.LONG, () -> strategy.newDynamicLongField(context, name) ); } else if (parseableAsDouble && context.root().numericDetection()) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.DOUBLE, @@ -90,22 +90,21 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam // failure to parse this, continue continue; } - createDynamicDateField( + return createDynamicDateField( context, name, dateTimeFormatter, () -> strategy.newDynamicDateField(context, name, dateTimeFormatter) ); - return; } - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.STRING, () -> strategy.newDynamicStringField(context, name) ); } else { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.STRING, @@ -117,7 +116,7 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG || numberType == XContentParser.NumberType.BIG_INTEGER) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.LONG, @@ -126,7 +125,7 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam } else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE || numberType == XContentParser.NumberType.BIG_DECIMAL) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.DOUBLE, @@ -136,7 +135,7 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam throw new IllegalStateException("Unable to parse number of type [" + numberType + "]"); } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.BOOLEAN, @@ -144,14 +143,14 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam ); } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { // runtime binary fields are not supported, hence binary objects always get created as concrete fields - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.BINARY, () -> CONCRETE.newDynamicBinaryField(context, name) ); } else { - createDynamicStringFieldFromTemplate(context, name); + return createDynamicStringFieldFromTemplate(context, name); } } @@ -178,40 +177,41 @@ static Mapper createObjectMapperFromTemplate(DocumentParserContext context, Stri * Creates a dynamic string field based on a matching dynamic template. * No field is created in case there is no matching dynamic template. */ - static void createDynamicStringFieldFromTemplate(DocumentParserContext context, String name) throws IOException { - createDynamicField(context, name, DynamicTemplate.XContentFieldType.STRING, () -> {}); + static boolean createDynamicStringFieldFromTemplate(DocumentParserContext context, String name) throws IOException { + return createDynamicField(context, name, DynamicTemplate.XContentFieldType.STRING, () -> false); } - private static void createDynamicDateField( + private static boolean createDynamicDateField( DocumentParserContext context, String name, DateFormatter dateFormatter, - CheckedRunnable createDynamicField + CheckedSupplier createDynamicField ) throws IOException { - createDynamicField(context, name, DynamicTemplate.XContentFieldType.DATE, dateFormatter, createDynamicField); + return createDynamicField(context, name, DynamicTemplate.XContentFieldType.DATE, dateFormatter, createDynamicField); } - private static void createDynamicField( + private static boolean createDynamicField( DocumentParserContext context, String name, DynamicTemplate.XContentFieldType matchType, - CheckedRunnable dynamicFieldStrategy + CheckedSupplier dynamicFieldStrategy ) throws IOException { assert matchType != DynamicTemplate.XContentFieldType.DATE; - createDynamicField(context, name, matchType, null, dynamicFieldStrategy); + return createDynamicField(context, name, matchType, null, dynamicFieldStrategy); } - private static void createDynamicField( + private static boolean createDynamicField( DocumentParserContext context, String name, DynamicTemplate.XContentFieldType matchType, DateFormatter dateFormatter, - CheckedRunnable dynamicFieldStrategy + CheckedSupplier dynamicFieldStrategy ) throws IOException { if (applyMatchingTemplate(context, name, matchType, dateFormatter)) { context.markFieldAsAppliedFromTemplate(name); + return true; } else { - dynamicFieldStrategy.run(); + return dynamicFieldStrategy.get(); } } @@ -284,15 +284,15 @@ private static Mapper.Builder parseDynamicTemplateMapping( * Defines how leaf fields of type string, long, double, boolean and date are dynamically mapped */ private interface Strategy { - void newDynamicStringField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException; - void newDynamicLongField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicLongField(DocumentParserContext context, String name) throws IOException; - void newDynamicDoubleField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicDoubleField(DocumentParserContext context, String name) throws IOException; - void newDynamicBooleanField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException; - void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException; + boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException; } /** @@ -307,25 +307,43 @@ private static final class Concrete implements Strategy { this.parseField = parseField; } - void createDynamicField(Mapper.Builder builder, DocumentParserContext context) throws IOException { - Mapper mapper = builder.build(context.createDynamicMapperBuilderContext()); - context.addDynamicMapper(mapper); - parseField.accept(context, mapper); + boolean createDynamicField(Mapper.Builder builder, DocumentParserContext context, MapperBuilderContext mapperBuilderContext) + throws IOException { + Mapper mapper = builder.build(mapperBuilderContext); + if (context.addDynamicMapper(mapper)) { + parseField.accept(context, mapper); + return true; + } else { + return false; + } + } + + boolean createDynamicField(Mapper.Builder builder, DocumentParserContext context) throws IOException { + return createDynamicField(builder, context, context.createDynamicMapperBuilderContext()); } @Override - public void newDynamicStringField(DocumentParserContext context, String name) throws IOException { - createDynamicField( - new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField( - new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) - ), - context - ); + public boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException { + MapperBuilderContext mapperBuilderContext = context.createDynamicMapperBuilderContext(); + if (mapperBuilderContext.parentObjectContainsDimensions()) { + return createDynamicField( + new KeywordFieldMapper.Builder(name, context.indexSettings().getIndexVersionCreated()), + context, + mapperBuilderContext + ); + } else { + return createDynamicField( + new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField( + new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) + ), + context + ); + } } @Override - public void newDynamicLongField(DocumentParserContext context, String name) throws IOException { - createDynamicField( + public boolean newDynamicLongField(DocumentParserContext context, String name) throws IOException { + return createDynamicField( new NumberFieldMapper.Builder( name, NumberFieldMapper.NumberType.LONG, @@ -339,11 +357,11 @@ public void newDynamicLongField(DocumentParserContext context, String name) thro } @Override - public void newDynamicDoubleField(DocumentParserContext context, String name) throws IOException { + public boolean newDynamicDoubleField(DocumentParserContext context, String name) throws IOException { // no templates are defined, we use float by default instead of double // since this is much more space-efficient and should be enough most of // the time - createDynamicField( + return createDynamicField( new NumberFieldMapper.Builder( name, NumberFieldMapper.NumberType.FLOAT, @@ -357,10 +375,10 @@ public void newDynamicDoubleField(DocumentParserContext context, String name) th } @Override - public void newDynamicBooleanField(DocumentParserContext context, String name) throws IOException { + public boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException { Settings settings = context.indexSettings().getSettings(); boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings); - createDynamicField( + return createDynamicField( new BooleanFieldMapper.Builder( name, ScriptCompiler.NONE, @@ -372,10 +390,10 @@ public void newDynamicBooleanField(DocumentParserContext context, String name) t } @Override - public void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateTimeFormatter) throws IOException { + public boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateTimeFormatter) throws IOException { Settings settings = context.indexSettings().getSettings(); boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings); - createDynamicField( + return createDynamicField( new DateFieldMapper.Builder( name, DateFieldMapper.Resolution.MILLISECONDS, @@ -388,8 +406,8 @@ public void newDynamicDateField(DocumentParserContext context, String name, Date ); } - void newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { - createDynamicField(new BinaryFieldMapper.Builder(name), context); + boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { + return createDynamicField(new BinaryFieldMapper.Builder(name), context); } } @@ -399,40 +417,43 @@ void newDynamicBinaryField(DocumentParserContext context, String name) throws IO * @see Dynamic */ private static final class Runtime implements Strategy { - static void createDynamicField(RuntimeField runtimeField, DocumentParserContext context) { - context.addDynamicRuntimeField(runtimeField); + static boolean createDynamicField(RuntimeField runtimeField, DocumentParserContext context) { + return context.addDynamicRuntimeField(runtimeField); } @Override - public void newDynamicStringField(DocumentParserContext context, String name) { + public boolean newDynamicStringField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(KeywordScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(KeywordScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicLongField(DocumentParserContext context, String name) { + public boolean newDynamicLongField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(LongScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(LongScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicDoubleField(DocumentParserContext context, String name) { + public boolean newDynamicDoubleField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(DoubleScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(DoubleScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicBooleanField(DocumentParserContext context, String name) { + public boolean newDynamicBooleanField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(BooleanScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(BooleanScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) { + public boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) { String fullName = context.path().pathAsText(name); MappingParserContext parserContext = context.dynamicTemplateParserContext(dateFormatter); - createDynamicField(DateScriptFieldType.sourceOnly(fullName, dateFormatter, parserContext.indexVersionCreated()), context); + return createDynamicField( + DateScriptFieldType.sourceOnly(fullName, dateFormatter, parserContext.indexVersionCreated()), + context + ); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 56a50c2dee0aa..8ce726b49ff66 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -166,6 +166,9 @@ protected Parameter[] getParameters() { @Override public IpFieldMapper build(MapperBuilderContext context) { + if (context.parentObjectContainsDimensions()) { + dimension.setValue(true); + } return new IpFieldMapper( name, new IpFieldType( @@ -555,7 +558,7 @@ private static InetAddress value(XContentParser parser, InetAddress nullValue) t private void indexValue(DocumentParserContext context, InetAddress address) { if (dimension) { - context.getDimensions().addIp(fieldType().name(), address); + context.getDimensions().addIp(fieldType().name(), address).validate(context.indexSettings()); } if (indexed) { Field field = new InetAddressPoint(fieldType().name(), address); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index c3a740e4cbfe6..a5a571fb82d85 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -242,6 +242,11 @@ public Builder dimension(boolean dimension) { return this; } + public Builder indexed(boolean indexed) { + this.indexed.setValue(indexed); + return this; + } + private FieldValues scriptValues() { if (script.get() == null) { return null; @@ -299,6 +304,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } + if (context.parentObjectContainsDimensions()) { + dimension(true); + } return new KeywordFieldType( context.buildFullName(name), fieldType, @@ -921,7 +929,7 @@ private void indexValue(DocumentParserContext context, String value) { final BytesRef binaryValue = new BytesRef(value); if (fieldType().isDimension()) { - context.getDimensions().addString(fieldType().name(), binaryValue); + context.getDimensions().addString(fieldType().name(), binaryValue).validate(context.indexSettings()); } // If the UTF8 encoding of the field value is bigger than the max length 32766, Lucene fill fail the indexing request and, to diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index 7506e8b8f6671..4154c936bab52 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -19,17 +19,23 @@ public class MapperBuilderContext { * The root context, to be used when building a tree of mappers */ public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream) { - return new MapperBuilderContext(null, isSourceSynthetic, isDataStream); + return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, false); } private final String path; private final boolean isSourceSynthetic; private final boolean isDataStream; + private final boolean parentObjectContainsDimensions; - MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream) { + MapperBuilderContext(String path) { + this(path, false, false, false); + } + + MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream, boolean parentObjectContainsDimensions) { this.path = path; this.isSourceSynthetic = isSourceSynthetic; this.isDataStream = isDataStream; + this.parentObjectContainsDimensions = parentObjectContainsDimensions; } /** @@ -38,7 +44,7 @@ public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDat * @return a new MapperBuilderContext with this context as its parent */ public MapperBuilderContext createChildContext(String name) { - return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream); + return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream, parentObjectContainsDimensions); } /** @@ -64,4 +70,12 @@ public boolean isSourceSynthetic() { public boolean isDataStream() { return isDataStream; } + + /** + * Are these field mappings being built dimensions? + */ + public boolean parentObjectContainsDimensions() { + return parentObjectContainsDimensions; + } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java index 79adaf5966c5b..0af182f315559 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java @@ -96,7 +96,6 @@ private Unlimited() {} public boolean decrementIfPossible(long fieldSize) { return true; } - } final class Limited implements NewFieldsBudget { @@ -115,7 +114,6 @@ public boolean decrementIfPossible(long fieldSize) { } return false; } - } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 20bd7cf4a87e6..9e47b42c4b18e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; @@ -118,6 +119,12 @@ public boolean isAutoUpdate() { Property.IndexScope, Property.ServerlessPublic ); + public static final Setting INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING = Setting.boolSetting( + "index.mapping.total_fields.ignore_dynamic_beyond_limit", + false, + Property.Dynamic, + Property.IndexScope + ); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting( "index.mapping.depth.limit", 20L, @@ -134,7 +141,7 @@ public boolean isAutoUpdate() { ); public static final Setting INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING = Setting.longSetting( "index.mapping.dimension_fields.limit", - 21, + 32_768, 0, Property.Dynamic, Property.IndexScope @@ -331,7 +338,7 @@ public void updateMapping(final IndexMetadata currentIndexMetadata, final IndexM } private boolean assertRefreshIsNotNeeded(DocumentMapper currentMapper, String type, Mapping incomingMapping) { - Mapping mergedMapping = mergeMappings(currentMapper, incomingMapping, MergeReason.MAPPING_RECOVERY); + Mapping mergedMapping = mergeMappings(currentMapper, incomingMapping, MergeReason.MAPPING_RECOVERY, indexSettings); // skip the runtime section or removed runtime fields will make the assertion fail ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(RootObjectMapper.TOXCONTENT_SKIP_RUNTIME, "true")); CompressedXContent mergedMappingSource; @@ -535,7 +542,7 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge private synchronized DocumentMapper doMerge(String type, MergeReason reason, Map mappingSourceAsMap) { Mapping incomingMapping = parseMapping(type, mappingSourceAsMap); - Mapping mapping = mergeMappings(this.mapper, incomingMapping, reason); + Mapping mapping = mergeMappings(this.mapper, incomingMapping, reason, this.indexSettings); // TODO: In many cases the source here is equal to mappingSource so we need not serialize again. // We should identify these cases reliably and save expensive serialization here DocumentMapper newMapper = newDocumentMapper(mapping, reason, mapping.toCompressedXContent()); @@ -576,8 +583,38 @@ public Mapping parseMapping(String mappingType, Map mappingSourc } } - public static Mapping mergeMappings(DocumentMapper currentMapper, Mapping incomingMapping, MergeReason reason) { - return mergeMappings(currentMapper, incomingMapping, reason, Long.MAX_VALUE); + public static Mapping mergeMappings( + DocumentMapper currentMapper, + Mapping incomingMapping, + MergeReason reason, + IndexSettings indexSettings + ) { + return mergeMappings(currentMapper, incomingMapping, reason, getMaxFieldsToAddDuringMerge(currentMapper, indexSettings, reason)); + } + + private static long getMaxFieldsToAddDuringMerge(DocumentMapper currentMapper, IndexSettings indexSettings, MergeReason reason) { + if (reason.isAutoUpdate() && indexSettings.isIgnoreDynamicFieldsBeyondLimit()) { + // If the index setting ignore_dynamic_beyond_limit is enabled, + // data nodes only add new dynamic fields until the limit is reached while parsing documents to be ingested. + // However, if there are concurrent mapping updates, + // data nodes may add dynamic fields under an outdated assumption that enough capacity is still available. + // When data nodes send the dynamic mapping update request to the master node, + // it will only add as many fields as there's actually capacity for when merging mappings. + long totalFieldsLimit = indexSettings.getMappingTotalFieldsLimit(); + return Optional.ofNullable(currentMapper) + .map(DocumentMapper::mappers) + .map(ml -> ml.remainingFieldsUntilLimit(totalFieldsLimit)) + .orElse(totalFieldsLimit); + } else { + // Else, we're not limiting the number of fields so that the merged mapping fails validation if it exceeds total_fields.limit. + // This is the desired behavior when making an explicit mapping update, even if ignore_dynamic_beyond_limit is enabled. + // When ignore_dynamic_beyond_limit is disabled and a dynamic mapping update would exceed the field limit, + // the document will get rejected. + // Normally, this happens on the data node in DocumentParserContext.addDynamicMapper but if there's a race condition, + // data nodes may add dynamic fields under an outdated assumption that enough capacity is still available. + // In this case, the master node will reject mapping updates that would exceed the limit when handling the mapping update. + return Long.MAX_VALUE; + } } static Mapping mergeMappings(DocumentMapper currentMapper, Mapping incomingMapping, MergeReason reason, long newFieldsBudget) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 0172c22c0b176..ea59d6640f647 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -282,7 +282,11 @@ void checkFieldLimit(long limit, int additionalFieldsToAdd) { } boolean exceedsLimit(long limit, int additionalFieldsToAdd) { - return getTotalFieldsCount() + additionalFieldsToAdd - mapping.getSortedMetadataMappers().length > limit; + return remainingFieldsUntilLimit(limit) < additionalFieldsToAdd; + } + + long remainingFieldsUntilLimit(long mappingTotalFieldsLimit) { + return mappingTotalFieldsLimit - getTotalFieldsCount() + mapping.getSortedMetadataMappers().length; } private void checkDimensionFieldLimit(long limit) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index 268d028be91a6..a654819811621 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -118,7 +118,7 @@ private static class NestedMapperBuilderContext extends MapperBuilderContext { final boolean parentIncludedInRoot; NestedMapperBuilderContext(String path, boolean parentIncludedInRoot) { - super(path, false, false); + super(path); this.parentIncludedInRoot = parentIncludedInRoot; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index d25832a28d318..5935eaf2c3d14 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -267,6 +267,10 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { + if (context.parentObjectContainsDimensions()) { + dimension.setValue(true); + } + MappedFieldType ft = new NumberFieldType(context.buildFullName(name), this); return new NumberFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); } @@ -1879,7 +1883,7 @@ public Number value(XContentParser parser) throws IllegalArgumentException, IOEx */ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { - context.getDimensions().addLong(fieldType().name(), numericValue.longValue()); + context.getDimensions().addLong(fieldType().name(), numericValue.longValue()).validate(context.indexSettings()); } fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 6ced2b49bb84a..9d7353859ed25 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -295,7 +295,10 @@ protected static void parseProperties( } } - if (objBuilder.subobjects.value() == false && type.equals(ObjectMapper.CONTENT_TYPE)) { + if (objBuilder.subobjects.value() == false + && (type.equals(ObjectMapper.CONTENT_TYPE) + || type.equals(NestedObjectMapper.CONTENT_TYPE) + || type.equals(PassThroughObjectMapper.CONTENT_TYPE))) { throw new MapperParsingException( "Tried to add subobject [" + fieldName @@ -304,21 +307,12 @@ protected static void parseProperties( + "] which does not support subobjects" ); } - if (objBuilder.subobjects.value() == false && type.equals(NestedObjectMapper.CONTENT_TYPE)) { - throw new MapperParsingException( - "Tried to add nested object [" - + fieldName - + "] to object [" - + objBuilder.name() - + "] which does not support subobjects" - ); - } Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]"); } Mapper.Builder fieldBuilder; - if (objBuilder.subobjects.value() == false) { + if (objBuilder.subobjects.value() == false || type.equals(FieldAliasMapper.CONTENT_TYPE)) { fieldBuilder = typeParser.parse(fieldName, propNode, parserContext); } else { String[] fieldNameParts = fieldName.split("\\."); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index bf540eae5ed49..0c604cb171457 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -35,7 +35,6 @@ public class ParsedDocument { private BytesReference source; private XContentType xContentType; - private Mapping dynamicMappingsUpdate; /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java new file mode 100644 index 0000000000000..b49c9328fcc79 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.Explicit; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; + +/** + * Mapper for pass-through objects. + * + * Pass-through objects allow creating fields inside them that can also be referenced directly in search queries. + * They also include parameters that affect how nested fields get configured. For instance, if parameter "time_series_dimension" + * is set, eligible subfields are marked as dimensions and keyword fields are additionally included in routing and tsid calculations. + */ +public class PassThroughObjectMapper extends ObjectMapper { + public static final String CONTENT_TYPE = "passthrough"; + + public static class Builder extends ObjectMapper.Builder { + + // Controls whether subfields are configured as time-series dimensions. + protected Explicit timeSeriesDimensionSubFields = Explicit.IMPLICIT_FALSE; + + public Builder(String name) { + // Subobjects are not currently supported. + super(name, Explicit.IMPLICIT_FALSE); + } + + @Override + public PassThroughObjectMapper.Builder add(Mapper.Builder builder) { + if (timeSeriesDimensionSubFields.value() && builder instanceof KeywordFieldMapper.Builder keywordBuilder) { + keywordBuilder.dimension(true); + } + super.add(builder); + return this; + } + + public PassThroughObjectMapper.Builder setContainsDimensions() { + timeSeriesDimensionSubFields = Explicit.EXPLICIT_TRUE; + return this; + } + + @Override + public PassThroughObjectMapper build(MapperBuilderContext context) { + return new PassThroughObjectMapper( + name, + context.buildFullName(name), + enabled, + dynamic, + buildMappers(context.createChildContext(name)), + timeSeriesDimensionSubFields + ); + } + } + + // If set, its subfields are marked as time-series dimensions (for the types supporting this). + private final Explicit timeSeriesDimensionSubFields; + + PassThroughObjectMapper( + String name, + String fullPath, + Explicit enabled, + Dynamic dynamic, + Map mappers, + Explicit timeSeriesDimensionSubFields + ) { + // Subobjects are not currently supported. + super(name, fullPath, enabled, Explicit.IMPLICIT_FALSE, dynamic, mappers); + this.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; + } + + @Override + PassThroughObjectMapper withoutMappers() { + return new PassThroughObjectMapper(simpleName(), fullPath(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields); + } + + public boolean containsDimensions() { + return timeSeriesDimensionSubFields.value(); + } + + @Override + public PassThroughObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { + PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(simpleName()); + builder.enabled = enabled; + builder.dynamic = dynamic; + builder.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; + return builder; + } + + public PassThroughObjectMapper merge(ObjectMapper mergeWith, MergeReason reason, MapperMergeContext parentBuilderContext) { + final var mergeResult = MergeResult.build(this, mergeWith, reason, parentBuilderContext); + PassThroughObjectMapper mergeWithObject = (PassThroughObjectMapper) mergeWith; + + final Explicit containsDimensions = (mergeWithObject.timeSeriesDimensionSubFields.explicit()) + ? mergeWithObject.timeSeriesDimensionSubFields + : this.timeSeriesDimensionSubFields; + + return new PassThroughObjectMapper( + simpleName(), + fullPath(), + mergeResult.enabled(), + mergeResult.dynamic(), + mergeResult.mappers(), + containsDimensions + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(simpleName()); + builder.field("type", CONTENT_TYPE); + if (timeSeriesDimensionSubFields.explicit()) { + builder.field(TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM, timeSeriesDimensionSubFields.value()); + } + if (dynamic != null) { + builder.field("dynamic", dynamic.name().toLowerCase(Locale.ROOT)); + } + if (isEnabled() != Defaults.ENABLED) { + builder.field("enabled", enabled.value()); + } + serializeMappers(builder, params); + return builder.endObject(); + } + + public static class TypeParser extends ObjectMapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, MappingParserContext parserContext) + throws MapperParsingException { + PassThroughObjectMapper.Builder builder = new Builder(name); + parsePassthrough(name, node, builder); + parseObjectFields(node, parserContext, builder); + return builder; + } + + protected static void parsePassthrough(String name, Map node, PassThroughObjectMapper.Builder builder) { + Object fieldNode = node.get(TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM); + if (fieldNode != null) { + builder.timeSeriesDimensionSubFields = Explicit.explicitBoolean( + nodeBooleanValue(fieldNode, name + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM) + ); + node.remove(TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 5d719ae4f5da7..7994c018f40f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -18,6 +18,8 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -43,6 +45,7 @@ public class RootObjectMapper extends ObjectMapper { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RootObjectMapper.class); + private static final int MAX_NESTING_LEVEL_FOR_PASS_THROUGH_OBJECTS = 20; /** * Parameter used when serializing {@link RootObjectMapper} and request that the runtime section is skipped. @@ -74,6 +77,8 @@ public static class Builder extends ObjectMapper.Builder { protected Explicit dateDetection = Defaults.DATE_DETECTION; protected Explicit numericDetection = Defaults.NUMERIC_DETECTION; + private static final Logger logger = LogManager.getLogger(RootObjectMapper.Builder.class); + public Builder(String name, Explicit subobjects) { super(name, subobjects); } @@ -106,12 +111,18 @@ public RootObjectMapper.Builder addRuntimeFields(Map runti @Override public RootObjectMapper build(MapperBuilderContext context) { + Map mappers = buildMappers(context); + + Map aliasMappers = new HashMap<>(); + getAliasMappers(mappers, aliasMappers, context, 0); + mappers.putAll(aliasMappers); + return new RootObjectMapper( name, enabled, subobjects, dynamic, - buildMappers(context), + mappers, new HashMap<>(runtimeFields), dynamicDateTimeFormatters, dynamicTemplates, @@ -119,6 +130,44 @@ public RootObjectMapper build(MapperBuilderContext context) { numericDetection ); } + + void getAliasMappers(Map mappers, Map aliasMappers, MapperBuilderContext context, int level) { + if (level >= MAX_NESTING_LEVEL_FOR_PASS_THROUGH_OBJECTS) { + logger.warn("Exceeded maximum nesting level for searching for pass-through object fields within object fields."); + return; + } + for (Mapper mapper : mappers.values()) { + // Create aliases for all fields in child passthrough mappers and place them under the root object. + if (mapper instanceof PassThroughObjectMapper passthroughMapper) { + for (Mapper internalMapper : passthroughMapper.mappers.values()) { + if (internalMapper instanceof FieldMapper fieldMapper) { + // If there's a conflicting alias with the same name at the root level, we don't want to throw an error + // to avoid indexing disruption. + // TODO: record an error without affecting document indexing, so that it can be investigated later. + Mapper conflict = mappers.get(fieldMapper.simpleName()); + if (conflict != null) { + if (conflict.typeName().equals(FieldAliasMapper.CONTENT_TYPE) == false + || ((FieldAliasMapper) conflict).path().equals(fieldMapper.mappedFieldType.name()) == false) { + logger.warn( + "Root alias for field " + + fieldMapper.name() + + " conflicts with existing field or alias, skipping alias creation." + ); + } + } else { + FieldAliasMapper aliasMapper = new FieldAliasMapper.Builder(fieldMapper.simpleName()).path( + fieldMapper.mappedFieldType.name() + ).build(context); + aliasMappers.put(aliasMapper.simpleName(), aliasMapper); + } + } + } + } else if (mapper instanceof ObjectMapper objectMapper) { + // Call recursively to check child fields. The level guards against long recursive call sequences. + getAliasMappers(objectMapper.mappers, aliasMappers, context, level + 1); + } + } + } } private final Explicit dynamicDateTimeFormatters; @@ -269,10 +318,8 @@ RootObjectMapper merge(RootObjectMapper mergeWithObject, MergeReason reason, Map runtimeFields.remove(runtimeField.getKey()); } else if (runtimeFields.containsKey(runtimeField.getKey())) { runtimeFields.put(runtimeField.getKey(), runtimeField.getValue()); - } else { - if (parentMergeContext.decrementFieldBudgetIfPossible(1)) { - runtimeFields.put(runtimeField.getValue().name(), runtimeField.getValue()); - } + } else if (parentMergeContext.decrementFieldBudgetIfPossible(1)) { + runtimeFields.put(runtimeField.getValue().name(), runtimeField.getValue()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 8e8c58b35c68a..2635c1c11be8e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -26,10 +26,8 @@ import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.script.field.SeqNoDocValuesField; -import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.Objects; /** * Mapper for the {@code _seq_no} field. @@ -58,28 +56,16 @@ private static class SingleValueLongField extends Field { FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } - private final BytesRef pointValue; - - SingleValueLongField(String field, long value) { + SingleValueLongField(String field) { super(field, FIELD_TYPE); - fieldsData = value; - pointValue = new BytesRef(new byte[Long.BYTES]); - assert pointValue.offset == 0; - assert pointValue.length == Long.BYTES; - NumericUtils.longToSortableBytes((Long) fieldsData, pointValue.bytes, 0); + fieldsData = SequenceNumbers.UNASSIGNED_SEQ_NO; } @Override public BytesRef binaryValue() { - return pointValue; - } - - @Override - public void setLongValue(long value) { - super.setLongValue(value); - assert pointValue.offset == 0; - assert pointValue.length == Long.BYTES; - NumericUtils.longToSortableBytes((Long) fieldsData, pointValue.bytes, 0); + final byte[] pointValue = new byte[Long.BYTES]; + NumericUtils.longToSortableBytes((Long) fieldsData, pointValue, 0); + return new BytesRef(pointValue); } @Override @@ -95,23 +81,21 @@ public String toString() { */ public static class SequenceIDFields { - private final Field seqNo; - private final Field primaryTerm; - private final Field tombstoneField; + private static final Field TOMBSTONE_FIELD = new NumericDocValuesField(TOMBSTONE_NAME, 1); + + private final Field seqNo = new SingleValueLongField(NAME); + private final Field primaryTerm = new NumericDocValuesField(PRIMARY_TERM_NAME, 0); + private final boolean isTombstone; - private SequenceIDFields(Field seqNo, Field primaryTerm, Field tombstoneField) { - Objects.requireNonNull(seqNo, "sequence number field cannot be null"); - Objects.requireNonNull(primaryTerm, "primary term field cannot be null"); - this.seqNo = seqNo; - this.primaryTerm = primaryTerm; - this.tombstoneField = tombstoneField; + private SequenceIDFields(boolean isTombstone) { + this.isTombstone = isTombstone; } public void addFields(LuceneDocument document) { document.add(seqNo); document.add(primaryTerm); - if (tombstoneField != null) { - document.add(tombstoneField); + if (isTombstone) { + document.add(TOMBSTONE_FIELD); } } @@ -129,19 +113,11 @@ public void set(long seqNo, long primaryTerm) { * calling {@link #set}. */ public static SequenceIDFields emptySeqID() { - return new SequenceIDFields( - new SingleValueLongField(NAME, SequenceNumbers.UNASSIGNED_SEQ_NO), - new NumericDocValuesField(PRIMARY_TERM_NAME, 0), - null - ); + return new SequenceIDFields(false); } public static SequenceIDFields tombstone() { - return new SequenceIDFields( - new SingleValueLongField(NAME, SequenceNumbers.UNASSIGNED_SEQ_NO), - new NumericDocValuesField(PRIMARY_TERM_NAME, 0), - new NumericDocValuesField(TOMBSTONE_NAME, 1) - ); + return new SequenceIDFields(true); } } @@ -249,23 +225,16 @@ private SeqNoFieldMapper() { } @Override - public void preParse(DocumentParserContext context) { + public void postParse(DocumentParserContext context) { // see InternalEngine.innerIndex to see where the real version value is set // also see ParsedDocument.updateSeqID (called by innerIndex) - SequenceIDFields seqID = SequenceIDFields.emptySeqID(); - context.seqID(seqID); - seqID.addFields(context.doc()); - } - - @Override - public void postParse(DocumentParserContext context) throws IOException { // In the case of nested docs, let's fill nested docs with the original // so that Lucene doesn't write a Bitset for documents that // don't have the field. This is consistent with the default value // for efficiency. // we share the parent docs fields to ensure good compression SequenceIDFields seqID = context.seqID(); - assert seqID != null; + seqID.addFields(context.doc()); for (LuceneDocument doc : context.nonRootDocuments()) { doc.add(seqID.seqNo); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index a85ba6d0e9a45..d7b8a9a49970e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -10,16 +10,22 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.search.Query; -import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.hash.Murmur3Hasher; +import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -33,12 +39,13 @@ import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; +import java.util.Base64; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; -import java.util.Locale; import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.SortedSet; +import java.util.TreeSet; /** * Mapper for {@code _tsid} field included generated when the index is @@ -50,24 +57,7 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { public static final String CONTENT_TYPE = "_tsid"; public static final TimeSeriesIdFieldType FIELD_TYPE = new TimeSeriesIdFieldType(); public static final TimeSeriesIdFieldMapper INSTANCE = new TimeSeriesIdFieldMapper(); - - /** - * The maximum length of the tsid. The value itself comes from a range check in - * Lucene's writer for utf-8 doc values. - */ - private static final int LIMIT = ByteBlockPool.BYTE_BLOCK_SIZE - 2; - /** - * Maximum length of the name of dimension. We picked this so that we could - * comfortable fit 16 dimensions inside {@link #LIMIT}. - */ - private static final int DIMENSION_NAME_LIMIT = 512; - /** - * The maximum length of any single dimension. We picked this so that we could - * comfortable fit 16 dimensions inside {@link #LIMIT}. This should be quite - * comfortable given that dimensions are typically going to be less than a - * hundred bytes each, but we're being paranoid here. - */ - private static final int DIMENSION_VALUE_LIMIT = 1024; + private static final Base64.Encoder BASE64_ENCODER = Base64.getUrlEncoder().withoutPadding(); @Override public FieldMapper.Builder getMergeBuilder() { @@ -144,12 +134,18 @@ private TimeSeriesIdFieldMapper() { public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; - TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); - BytesRef timeSeriesId = timeSeriesIdBuilder.build().toBytesRef(); + final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); + final BytesRef timeSeriesId = getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ID_HASHING) + ? timeSeriesIdBuilder.buildLegacyTsid().toBytesRef() + : timeSeriesIdBuilder.buildTsidHash().toBytesRef(); context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); } + private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { + return context.indexSettings().getIndexVersionCreated(); + } + @Override protected String contentType() { return CONTENT_TYPE; @@ -163,40 +159,30 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { /** * Decode the {@code _tsid} into a human readable map. */ - public static Map decodeTsid(StreamInput in) { + public static Object encodeTsid(StreamInput in) { try { - int size = in.readVInt(); - Map result = new LinkedHashMap(size); - - for (int i = 0; i < size; i++) { - String name = in.readBytesRef().utf8ToString(); - - int type = in.read(); - switch (type) { - case (byte) 's' -> // parse a string - result.put(name, in.readBytesRef().utf8ToString()); - case (byte) 'l' -> // parse a long - result.put(name, in.readLong()); - case (byte) 'u' -> { // parse an unsigned_long - Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.readLong()); - result.put(name, ul); - } - default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]"); - } - } - return result; - } catch (IOException | IllegalArgumentException e) { - throw new IllegalArgumentException("Error formatting " + NAME + ": " + e.getMessage(), e); + return base64Encode(in.readBytesRef()); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to read tsid"); } } public static class TimeSeriesIdBuilder implements DocumentDimensions { + + private static final int SEED = 0; + + public static final int MAX_DIMENSIONS = 512; + + private record Dimension(BytesRef name, BytesReference value) {} + + private final Murmur3Hasher tsidHasher = new Murmur3Hasher(0); + /** - * A sorted map of the serialized values of dimension fields that will be used + * A sorted set of the serialized values of dimension fields that will be used * for generating the _tsid field. The map will be used by {@link TimeSeriesIdFieldMapper} * to build the _tsid field for the document. */ - private final SortedMap dimensions = new TreeMap<>(); + private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ @@ -207,39 +193,88 @@ public TimeSeriesIdBuilder(@Nullable IndexRouting.ExtractFromSource.Builder rout this.routingBuilder = routingBuilder; } - public BytesReference build() throws IOException { + public BytesReference buildLegacyTsid() throws IOException { if (dimensions.isEmpty()) { throw new IllegalArgumentException("Dimension fields are missing."); } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); - for (Map.Entry entry : dimensions.entrySet()) { - BytesRef fieldName = entry.getKey(); - if (fieldName.length > DIMENSION_NAME_LIMIT) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Dimension name must be less than [%d] bytes but [%s] was [%s].", - DIMENSION_NAME_LIMIT, - fieldName.utf8ToString(), - fieldName.length - ) - ); - } - out.writeBytesRef(fieldName); - entry.getValue().writeTo(out); + for (Dimension entry : dimensions) { + out.writeBytesRef(entry.name); + entry.value.writeTo(out); } - BytesReference timeSeriesId = out.bytes(); - if (timeSeriesId.length() > LIMIT) { - throw new IllegalArgumentException(NAME + " longer than [" + LIMIT + "] bytes [" + timeSeriesId.length() + "]."); + return out.bytes(); + } + } + + /** + * Here we build the hash of the tsid using a similarity function so that we have a result + * with the following pattern: + * + * hash128(catenate(dimension field names)) + + * foreach(dimension field value, limit = MAX_DIMENSIONS) { hash32(dimension field value) } + + * hash128(catenate(dimension field values)) + * + * The idea is to be able to place 'similar' time series close to each other. Two time series + * are considered 'similar' if they share the same dimensions (names and values). + */ + public BytesReference buildTsidHash() throws IOException { + // NOTE: hash all dimension field names + int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); + int tsidHashIndex = 0; + byte[] tsidHash = new byte[16 + 16 + 4 * numberOfDimensions]; + + tsidHasher.reset(); + for (final Dimension dimension : dimensions) { + tsidHasher.update(dimension.name.bytes); + } + tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); + + // NOTE: concatenate all dimension value hashes up to a certain number of dimensions + int tsidHashStartIndex = tsidHashIndex; + for (final Dimension dimension : dimensions) { + if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) { + break; } - return timeSeriesId; + final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); + ByteUtils.writeIntLE( + StringHelper.murmurhash3_x86_32( + dimensionValueBytesRef.bytes, + dimensionValueBytesRef.offset, + dimensionValueBytesRef.length, + SEED + ), + tsidHash, + tsidHashIndex + ); + tsidHashIndex += 4; } + + // NOTE: hash all dimension field allValues + tsidHasher.reset(); + for (final Dimension dimension : dimensions) { + tsidHasher.update(dimension.value.toBytesRef().bytes); + } + tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); + + assert tsidHashIndex == tsidHash.length; + try (BytesStreamOutput out = new BytesStreamOutput(tsidHash.length)) { + out.writeBytesRef(new BytesRef(tsidHash, 0, tsidHash.length)); + return out.bytes(); + } + } + + private int writeHash128(final MurmurHash3.Hash128 hash128, byte[] buffer, int tsidHashIndex) { + ByteUtils.writeLongLE(hash128.h1, buffer, tsidHashIndex); + tsidHashIndex += 8; + ByteUtils.writeLongLE(hash128.h2, buffer, tsidHashIndex); + tsidHashIndex += 8; + return tsidHashIndex; } @Override - public void addString(String fieldName, BytesRef utf8Value) { + public DocumentDimensions addString(String fieldName, BytesRef utf8Value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 's'); /* @@ -247,11 +282,6 @@ public void addString(String fieldName, BytesRef utf8Value) { * so it's easier for folks to reason about the space taken up. Mostly * it'll be smaller too. */ - if (utf8Value.length > DIMENSION_VALUE_LIMIT) { - throw new IllegalArgumentException( - "Dimension fields must be less than [" + DIMENSION_VALUE_LIMIT + "] bytes but was [" + utf8Value.length + "]." - ); - } out.writeBytesRef(utf8Value); add(fieldName, out.bytes()); @@ -261,15 +291,16 @@ public void addString(String fieldName, BytesRef utf8Value) { } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } + return this; } @Override - public void addIp(String fieldName, InetAddress value) { - addString(fieldName, NetworkAddress.format(value)); + public DocumentDimensions addIp(String fieldName, InetAddress value) { + return addString(fieldName, NetworkAddress.format(value)); } @Override - public void addLong(String fieldName, long value) { + public DocumentDimensions addLong(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 'l'); out.writeLong(value); @@ -277,10 +308,11 @@ public void addLong(String fieldName, long value) { } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } + return this; } @Override - public void addUnsignedLong(String fieldName, long value) { + public DocumentDimensions addUnsignedLong(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value); if (ul instanceof Long l) { @@ -291,24 +323,89 @@ public void addUnsignedLong(String fieldName, long value) { out.writeLong(value); } add(fieldName, out.bytes()); + return this; } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } } - private void add(String fieldName, BytesReference encoded) { - BytesReference old = dimensions.put(new BytesRef(fieldName), encoded); - if (old != null) { + @Override + public DocumentDimensions validate(final IndexSettings settings) { + if (settings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ID_HASHING) + && dimensions.size() > settings.getValue(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING)) { + throw new MapperException( + "Too many dimension fields [" + + dimensions.size() + + "], max [" + + settings.getValue(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING) + + "] dimension fields allowed" + ); + } + return this; + } + + private void add(String fieldName, BytesReference encoded) throws IOException { + final Dimension dimension = new Dimension(new BytesRef(fieldName), encoded); + if (dimensions.contains(dimension)) { throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); } + dimensions.add(dimension); } } - public static Map decodeTsid(BytesRef bytesRef) { + public static Object encodeTsid(final BytesRef bytesRef) { + return base64Encode(bytesRef); + } + + private static String base64Encode(final BytesRef bytesRef) { + byte[] bytes = new byte[bytesRef.length]; + System.arraycopy(bytesRef.bytes, 0, bytes, 0, bytesRef.length); + return BASE64_ENCODER.encodeToString(bytes); + } + + public static Map decodeTsidAsMap(BytesRef bytesRef) { try (StreamInput input = new BytesArray(bytesRef).streamInput()) { - return decodeTsid(input); + return decodeTsidAsMap(input); } catch (IOException ex) { throw new IllegalArgumentException("Dimension field cannot be deserialized.", ex); } } + + public static Map decodeTsidAsMap(StreamInput in) { + try { + int size = in.readVInt(); + Map result = new LinkedHashMap<>(size); + + for (int i = 0; i < size; i++) { + String name = null; + try { + name = in.readBytesRef().utf8ToString(); + } catch (AssertionError ae) { + throw new IllegalArgumentException("Error parsing keyword dimension: " + ae.getMessage(), ae); + } + + int type = in.read(); + switch (type) { + case (byte) 's' -> { + // parse a string + try { + result.put(name, in.readBytesRef().utf8ToString()); + } catch (AssertionError ae) { + throw new IllegalArgumentException("Error parsing keyword dimension: " + ae.getMessage(), ae); + } + } + case (byte) 'l' -> // parse a long + result.put(name, in.readLong()); + case (byte) 'u' -> { // parse an unsigned_long + Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.readLong()); + result.put(name, ul); + } + default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]"); + } + } + return result; + } catch (IOException | IllegalArgumentException e) { + throw new IllegalArgumentException("Error formatting " + NAME + ": " + e.getMessage(), e); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 62bd8ec994639..27e281ed7fb52 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -42,6 +42,8 @@ public class TsidExtractingIdFieldMapper extends IdFieldMapper { public static final TsidExtractingIdFieldMapper INSTANCE = new TsidExtractingIdFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(MappingParserContext::idFieldMapper); + // NOTE: we use a prefix when hashing the tsid field so to be able later on (for instance for debugging purposes) + // to query documents whose tsid has been hashed. Using a prefix allows us to query using the prefix. static final class IdFieldType extends TermBasedFieldType { IdFieldType() { @@ -124,9 +126,6 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra * it always must pass. */ IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - assert context.getDynamicMappers().isEmpty() == false - || context.getDynamicRuntimeFields().isEmpty() == false - || id.equals(indexRouting.createId(TimeSeriesIdFieldMapper.decodeTsid(tsid), suffix)); assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); @@ -186,7 +185,7 @@ public String documentDescription(DocumentParserContext context) { StringBuilder description = new StringBuilder("a time series document"); IndexableField tsidField = context.doc().getField(TimeSeriesIdFieldMapper.NAME); if (tsidField != null) { - description.append(" with dimensions ").append(tsidDescription(tsidField)); + description.append(" with tsid ").append(tsidDescription(tsidField)); } IndexableField timestampField = context.doc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); if (timestampField != null) { @@ -205,7 +204,7 @@ public String documentDescription(ParsedDocument parsedDocument) { } private static String tsidDescription(IndexableField tsidField) { - String tsid = TimeSeriesIdFieldMapper.decodeTsid(tsidField.binaryValue()).toString(); + String tsid = TimeSeriesIdFieldMapper.encodeTsid(tsidField.binaryValue()).toString(); if (tsid.length() <= DESCRIPTION_TSID_LIMIT) { return tsid; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index f09c6f8c036c8..d373d683b73ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -173,7 +173,7 @@ private void addField(DocumentParserContext context, ContentPath path, String cu final String keyedFieldName = FlattenedFieldParser.extractKey(bytesKeyedValue).utf8ToString(); if (fieldType.isDimension() && fieldType.dimensions().contains(keyedFieldName)) { final BytesRef keyedFieldValue = FlattenedFieldParser.extractValue(bytesKeyedValue); - context.getDimensions().addString(rootFieldName + "." + keyedFieldName, keyedFieldValue); + context.getDimensions().addString(rootFieldName + "." + keyedFieldName, keyedFieldValue).validate(context.indexSettings()); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index a9a31ba585177..d36ca9e0b25c1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -46,6 +46,8 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat; +import org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.ArraySourceValueFetcher; @@ -842,6 +844,25 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new Int8HnswIndexOptions(m, efConstruction, confidenceInterval); } + }, + FLAT("flat") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new FlatIndexOptions(); + } + }, + INT8_FLAT("int8_flat") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); + Float confidenceInterval = null; + if (confidenceIntervalNode != null) { + confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode); + } + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new Int8FlatIndexOption(confidenceInterval); + } }; static Optional fromString(String type) { @@ -857,6 +878,80 @@ static Optional fromString(String type) { abstract IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap); } + private static class Int8FlatIndexOption extends IndexOptions { + private final Float confidenceInterval; + + Int8FlatIndexOption(Float confidenceInterval) { + super("int8_flat"); + this.confidenceInterval = confidenceInterval; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + if (confidenceInterval != null) { + builder.field("confidence_interval", confidenceInterval); + } + builder.endObject(); + return builder; + } + + @Override + KnnVectorsFormat getVectorsFormat() { + return new ES813Int8FlatVectorFormat(confidenceInterval); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Int8FlatIndexOption that = (Int8FlatIndexOption) o; + return Objects.equals(confidenceInterval, that.confidenceInterval); + } + + @Override + public int hashCode() { + return Objects.hash(confidenceInterval); + } + + @Override + boolean supportsElementType(ElementType elementType) { + return elementType != ElementType.BYTE; + } + } + + private static class FlatIndexOptions extends IndexOptions { + + FlatIndexOptions() { + super("flat"); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.endObject(); + return builder; + } + + @Override + KnnVectorsFormat getVectorsFormat() { + return new ES813FlatVectorFormat(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + return o != null && getClass() == o.getClass(); + } + + @Override + public int hashCode() { + return Objects.hash(type); + } + } + private static class Int8HnswIndexOptions extends IndexOptions { private final int m; private final int efConstruction; @@ -1186,7 +1281,6 @@ && isNotUnitVector(squaredMagnitude)) { case FLOAT -> parentFilter != null ? new ESDiversifyingChildrenFloatKnnVectorQuery(name(), queryVector, filter, numCands, parentFilter) : new ESKnnFloatVectorQuery(name(), queryVector, numCands, filter); - }; if (similarityThreshold != null) { diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index c04182dfacd54..86af6d21b7a09 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -59,6 +59,7 @@ import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.XContentParserConfiguration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -73,6 +74,7 @@ import java.util.function.Predicate; import static org.elasticsearch.index.IndexService.parseRuntimeMappings; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; /** * The context used to execute a search request on a shard. It provides access @@ -100,6 +102,8 @@ public class SearchExecutionContext extends QueryRewriteContext { private QueryBuilder aliasFilter; private boolean rewriteToNamedQueries = false; + private Integer requestSize = DEFAULT_SIZE; + /** * Build a {@linkplain SearchExecutionContext}. */ @@ -123,6 +127,52 @@ public SearchExecutionContext( BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, Map runtimeMappings + ) { + this( + shardId, + shardRequestIndex, + indexSettings, + bitsetFilterCache, + indexFieldDataLookup, + mapperService, + mappingLookup, + similarityService, + scriptService, + parserConfiguration, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + clusterAlias, + indexNameMatcher, + allowExpensiveQueries, + valuesSourceRegistry, + runtimeMappings, + null + ); + } + + public SearchExecutionContext( + int shardId, + int shardRequestIndex, + IndexSettings indexSettings, + BitsetFilterCache bitsetFilterCache, + BiFunction> indexFieldDataLookup, + MapperService mapperService, + MappingLookup mappingLookup, + SimilarityService similarityService, + ScriptCompiler scriptService, + XContentParserConfiguration parserConfiguration, + NamedWriteableRegistry namedWriteableRegistry, + Client client, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Predicate indexNameMatcher, + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings, + Integer requestSize ) { this( shardId, @@ -147,7 +197,8 @@ public SearchExecutionContext( allowExpensiveQueries, valuesSourceRegistry, parseRuntimeMappings(runtimeMappings, mapperService, indexSettings, mappingLookup), - null + null, + requestSize ); } @@ -172,7 +223,8 @@ public SearchExecutionContext(SearchExecutionContext source) { source.allowExpensiveQueries, source.getValuesSourceRegistry(), source.runtimeMappings, - source.allowedFields + source.allowedFields, + source.requestSize ); } @@ -196,7 +248,8 @@ private SearchExecutionContext( BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, Map runtimeMappings, - Predicate allowedFields + Predicate allowedFields, + Integer requestSize ) { super( parserConfig, @@ -221,6 +274,7 @@ private SearchExecutionContext( this.indexFieldDataLookup = indexFieldDataLookup; this.nestedScope = new NestedScope(); this.searcher = searcher; + this.requestSize = requestSize; } private void reset() { @@ -334,6 +388,19 @@ public boolean isMultiField(String field) { return mapperService.isMultiField(field); } + public Iterable dimensionFields() { + List dimensionFields = new ArrayList<>(); + for (var mapper : mapperService.mappingLookup().fieldMappers()) { + if (mapper instanceof FieldMapper fieldMapper) { + var fieldType = fieldMapper.fieldType(); + if (fieldType.isDimension()) { + dimensionFields.add(fieldType); + } + } + } + return dimensionFields; + } + public Set sourcePath(String fullName) { return mappingLookup.sourcePaths(fullName); } @@ -594,6 +661,10 @@ public IndexSearcher searcher() { return searcher; } + public Integer requestSize() { + return requestSize; + } + /** * Is this field present in the underlying lucene index for the current shard? */ diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java index ffafc1be6a7ba..0aeb64d5b250f 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java @@ -8,7 +8,7 @@ package org.elasticsearch.index.reindex; -import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestLazyBuilder; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; @@ -16,20 +16,44 @@ import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.DEFAULT_SCROLL_SIZE; +import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.DEFAULT_SCROLL_TIMEOUT; public abstract class AbstractBulkByScrollRequestBuilder< Request extends AbstractBulkByScrollRequest, - Self extends AbstractBulkByScrollRequestBuilder> extends ActionRequestBuilder { + Self extends AbstractBulkByScrollRequestBuilder> extends ActionRequestLazyBuilder { private final SearchRequestBuilder source; + private Integer maxDocs; + private Boolean abortOnVersionConflict; + private Boolean refresh; + private TimeValue timeout; + private ActiveShardCount waitForActiveShards; + private TimeValue retryBackoffInitialTime; + private Integer maxRetries; + private Float requestsPerSecond; + private Boolean shouldStoreResult; + private Integer slices; protected AbstractBulkByScrollRequestBuilder( ElasticsearchClient client, ActionType action, - SearchRequestBuilder source, - Request request + SearchRequestBuilder source ) { - super(client, action, request); + super(client, action); this.source = source; + initSourceSearchRequest(); + } + + /* + * The following is normally done within the AbstractBulkByScrollRequest constructor. But that constructor is not called until the + * request() method is called once this builder is complete. Doing it there blows away changes made to the source request. + */ + private void initSourceSearchRequest() { + source.request().scroll(DEFAULT_SCROLL_TIMEOUT); + source.request().source(new SearchSourceBuilder()); + source.request().source().size(DEFAULT_SCROLL_SIZE); } protected abstract Self self(); @@ -73,7 +97,7 @@ public Self size(int size) { * documents. */ public Self maxDocs(int maxDocs) { - request.setMaxDocs(maxDocs); + this.maxDocs = maxDocs; return self(); } @@ -81,7 +105,7 @@ public Self maxDocs(int maxDocs) { * Set whether or not version conflicts cause the action to abort. */ public Self abortOnVersionConflict(boolean abortOnVersionConflict) { - request.setAbortOnVersionConflict(abortOnVersionConflict); + this.abortOnVersionConflict = abortOnVersionConflict; return self(); } @@ -89,7 +113,7 @@ public Self abortOnVersionConflict(boolean abortOnVersionConflict) { * Call refresh on the indexes we've written to after the request ends? */ public Self refresh(boolean refresh) { - request.setRefresh(refresh); + this.refresh = refresh; return self(); } @@ -97,7 +121,7 @@ public Self refresh(boolean refresh) { * Timeout to wait for the shards on to be available for each bulk request. */ public Self timeout(TimeValue timeout) { - request.setTimeout(timeout); + this.timeout = timeout; return self(); } @@ -106,7 +130,7 @@ public Self timeout(TimeValue timeout) { * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details. */ public Self waitForActiveShards(ActiveShardCount activeShardCount) { - request.setWaitForActiveShards(activeShardCount); + this.waitForActiveShards = activeShardCount; return self(); } @@ -115,7 +139,7 @@ public Self waitForActiveShards(ActiveShardCount activeShardCount) { * is about one minute per bulk request. Once the entire bulk request is successful the retry counter resets. */ public Self setRetryBackoffInitialTime(TimeValue retryBackoffInitialTime) { - request.setRetryBackoffInitialTime(retryBackoffInitialTime); + this.retryBackoffInitialTime = retryBackoffInitialTime; return self(); } @@ -123,7 +147,7 @@ public Self setRetryBackoffInitialTime(TimeValue retryBackoffInitialTime) { * Total number of retries attempted for rejections. There is no way to ask for unlimited retries. */ public Self setMaxRetries(int maxRetries) { - request.setMaxRetries(maxRetries); + this.maxRetries = maxRetries; return self(); } @@ -133,7 +157,7 @@ public Self setMaxRetries(int maxRetries) { * make sure that it contains any time that we might wait. */ public Self setRequestsPerSecond(float requestsPerSecond) { - request.setRequestsPerSecond(requestsPerSecond); + this.requestsPerSecond = requestsPerSecond; return self(); } @@ -141,7 +165,7 @@ public Self setRequestsPerSecond(float requestsPerSecond) { * Should this task store its result after it has finished? */ public Self setShouldStoreResult(boolean shouldStoreResult) { - request.setShouldStoreResult(shouldStoreResult); + this.shouldStoreResult = shouldStoreResult; return self(); } @@ -149,7 +173,40 @@ public Self setShouldStoreResult(boolean shouldStoreResult) { * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks. */ public Self setSlices(int slices) { - request.setSlices(slices); + this.slices = slices; return self(); } + + protected void apply(Request request) { + if (maxDocs != null) { + request.setMaxDocs(maxDocs); + } + if (abortOnVersionConflict != null) { + request.setAbortOnVersionConflict(abortOnVersionConflict); + } + if (refresh != null) { + request.setRefresh(refresh); + } + if (timeout != null) { + request.setTimeout(timeout); + } + if (waitForActiveShards != null) { + request.setWaitForActiveShards(waitForActiveShards); + } + if (retryBackoffInitialTime != null) { + request.setRetryBackoffInitialTime(retryBackoffInitialTime); + } + if (maxRetries != null) { + request.setMaxRetries(maxRetries); + } + if (requestsPerSecond != null) { + request.setRequestsPerSecond(requestsPerSecond); + } + if (shouldStoreResult != null) { + request.setShouldStoreResult(shouldStoreResult); + } + if (slices != null) { + request.setSlices(slices); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java index 53e878b643517..30114b1472dd5 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java @@ -16,21 +16,29 @@ public abstract class AbstractBulkIndexByScrollRequestBuilder< Request extends AbstractBulkIndexByScrollRequest, Self extends AbstractBulkIndexByScrollRequestBuilder> extends AbstractBulkByScrollRequestBuilder { + private Script script; protected AbstractBulkIndexByScrollRequestBuilder( ElasticsearchClient client, ActionType action, - SearchRequestBuilder search, - Request request + SearchRequestBuilder search ) { - super(client, action, search, request); + super(client, action, search); } /** * Script to modify the documents before they are processed. */ public Self script(Script script) { - request.setScript(script); + this.script = script; return self(); } + + @Override + public void apply(Request request) { + super.apply(request); + if (script != null) { + request.setScript(script); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java index 85424c2eef7d2..6243859ec0e33 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java @@ -60,7 +60,7 @@ public DeleteByQueryRequest(StreamInput in) throws IOException { super(in); } - private DeleteByQueryRequest(SearchRequest search, boolean setDefaults) { + DeleteByQueryRequest(SearchRequest search, boolean setDefaults) { super(search, setDefaults); // Delete-By-Query does not require the source if (setDefaults) { diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java index 49d3c660a4b68..3452c6659a392 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java @@ -8,17 +8,21 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; public class DeleteByQueryRequestBuilder extends AbstractBulkByScrollRequestBuilder { + private Boolean abortOnVersionConflict; + public DeleteByQueryRequestBuilder(ElasticsearchClient client) { this(client, new SearchRequestBuilder(client)); } private DeleteByQueryRequestBuilder(ElasticsearchClient client, SearchRequestBuilder search) { - super(client, DeleteByQueryAction.INSTANCE, search, new DeleteByQueryRequest(search.request())); + super(client, DeleteByQueryAction.INSTANCE, search); + source().setFetchSource(false); } @Override @@ -28,7 +32,33 @@ protected DeleteByQueryRequestBuilder self() { @Override public DeleteByQueryRequestBuilder abortOnVersionConflict(boolean abortOnVersionConflict) { - request.setAbortOnVersionConflict(abortOnVersionConflict); + this.abortOnVersionConflict = abortOnVersionConflict; return this; } + + @Override + public DeleteByQueryRequest request() { + SearchRequest search = source().request(); + try { + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(search, false); + try { + apply(deleteByQueryRequest); + return deleteByQueryRequest; + } catch (Exception e) { + deleteByQueryRequest.decRef(); + throw e; + } + } catch (Exception e) { + search.decRef(); + throw e; + } + } + + @Override + public void apply(DeleteByQueryRequest request) { + super.apply(request); + if (abortOnVersionConflict != null) { + request.setAbortOnVersionConflict(abortOnVersionConflict); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index a1f741d7d51d6..683ec75c57d76 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -68,7 +68,7 @@ public ReindexRequest() { this(search, destination, true); } - private ReindexRequest(SearchRequest search, IndexRequest destination, boolean setDefaults) { + ReindexRequest(SearchRequest search, IndexRequest destination, boolean setDefaults) { super(search, setDefaults); this.destination = destination; } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java index 88a851bee15e0..156b39d608654 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java @@ -8,20 +8,23 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; public class ReindexRequestBuilder extends AbstractBulkIndexByScrollRequestBuilder { - private final IndexRequestBuilder destination; + private final IndexRequestBuilder destinationBuilder; + private RemoteInfo remoteInfo; public ReindexRequestBuilder(ElasticsearchClient client) { this(client, new SearchRequestBuilder(client), new IndexRequestBuilder(client)); } private ReindexRequestBuilder(ElasticsearchClient client, SearchRequestBuilder search, IndexRequestBuilder destination) { - super(client, ReindexAction.INSTANCE, search, new ReindexRequest(search.request(), destination.request())); - this.destination = destination; + super(client, ReindexAction.INSTANCE, search); + this.destinationBuilder = destination; } @Override @@ -30,14 +33,14 @@ protected ReindexRequestBuilder self() { } public IndexRequestBuilder destination() { - return destination; + return destinationBuilder; } /** * Set the destination index. */ public ReindexRequestBuilder destination(String index) { - destination.setIndex(index); + destinationBuilder.setIndex(index); return this; } @@ -45,7 +48,34 @@ public ReindexRequestBuilder destination(String index) { * Setup reindexing from a remote cluster. */ public ReindexRequestBuilder setRemoteInfo(RemoteInfo remoteInfo) { - request().setRemoteInfo(remoteInfo); + this.remoteInfo = remoteInfo; return this; } + + @Override + public ReindexRequest request() { + SearchRequest source = source().request(); + try { + IndexRequest destination = destinationBuilder.request(); + try { + ReindexRequest reindexRequest = new ReindexRequest(source, destination, false); + try { + super.apply(reindexRequest); + if (remoteInfo != null) { + reindexRequest.setRemoteInfo(remoteInfo); + } + return reindexRequest; + } catch (Exception e) { + reindexRequest.decRef(); + throw e; + } + } catch (Exception e) { + destination.decRef(); + throw e; + } + } catch (Exception e) { + source.decRef(); + throw e; + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java index d30b54fdafd42..44b959074ed76 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java @@ -52,7 +52,7 @@ public UpdateByQueryRequest(StreamInput in) throws IOException { pipeline = in.readOptionalString(); } - private UpdateByQueryRequest(SearchRequest search, boolean setDefaults) { + UpdateByQueryRequest(SearchRequest search, boolean setDefaults) { super(search, setDefaults); } diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java index b63ebdf1def86..270014d6ab3f2 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; @@ -15,12 +16,15 @@ public class UpdateByQueryRequestBuilder extends AbstractBulkIndexByScrollReques UpdateByQueryRequest, UpdateByQueryRequestBuilder> { + private Boolean abortOnVersionConflict; + private String pipeline; + public UpdateByQueryRequestBuilder(ElasticsearchClient client) { this(client, new SearchRequestBuilder(client)); } private UpdateByQueryRequestBuilder(ElasticsearchClient client, SearchRequestBuilder search) { - super(client, UpdateByQueryAction.INSTANCE, search, new UpdateByQueryRequest(search.request())); + super(client, UpdateByQueryAction.INSTANCE, search); } @Override @@ -30,12 +34,41 @@ protected UpdateByQueryRequestBuilder self() { @Override public UpdateByQueryRequestBuilder abortOnVersionConflict(boolean abortOnVersionConflict) { - request.setAbortOnVersionConflict(abortOnVersionConflict); + this.abortOnVersionConflict = abortOnVersionConflict; return this; } public UpdateByQueryRequestBuilder setPipeline(String pipeline) { - request.setPipeline(pipeline); + this.pipeline = pipeline; return this; } + + @Override + public UpdateByQueryRequest request() { + SearchRequest search = source().request(); + try { + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(search, false); + try { + apply(updateByQueryRequest); + return updateByQueryRequest; + } catch (Exception e) { + updateByQueryRequest.decRef(); + throw e; + } + } catch (Exception e) { + search.decRef(); + throw e; + } + } + + @Override + public void apply(UpdateByQueryRequest request) { + super.apply(request); + if (abortOnVersionConflict != null) { + request.setAbortOnVersionConflict(abortOnVersionConflict); + } + if (pipeline != null) { + request.setPipeline(pipeline); + } + } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 181852c2c3bc9..795ed2120b098 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -52,6 +52,7 @@ import org.elasticsearch.index.mapper.NestedPathFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.RuntimeField; @@ -193,6 +194,7 @@ public static Map getMappers(List mappe mappers.put(KeywordFieldMapper.CONTENT_TYPE, KeywordFieldMapper.PARSER); mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); mappers.put(NestedObjectMapper.CONTENT_TYPE, new NestedObjectMapper.TypeParser()); + mappers.put(PassThroughObjectMapper.CONTENT_TYPE, new PassThroughObjectMapper.TypeParser()); mappers.put(TextFieldMapper.CONTENT_TYPE, TextFieldMapper.PARSER); mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER); diff --git a/server/src/main/java/org/elasticsearch/inference/ChunkedInferenceServiceResults.java b/server/src/main/java/org/elasticsearch/inference/ChunkedInferenceServiceResults.java new file mode 100644 index 0000000000000..5ba2196e91488 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/inference/ChunkedInferenceServiceResults.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.inference; + +public interface ChunkedInferenceServiceResults extends InferenceServiceResults { + +} diff --git a/server/src/main/java/org/elasticsearch/inference/ChunkingOptions.java b/server/src/main/java/org/elasticsearch/inference/ChunkingOptions.java new file mode 100644 index 0000000000000..917cc68059136 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/inference/ChunkingOptions.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.inference; + +import org.elasticsearch.core.Nullable; + +public record ChunkingOptions(@Nullable Integer windowSize, @Nullable Integer span) { + + public boolean settingsArePresent() { + return windowSize != null || span != null; + } +} diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index fdeb32de33877..26c8eac53b0fb 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -76,6 +76,7 @@ default void init(Client client) {} * @param model The model * @param input Inference input * @param taskSettings Settings in the request to override the model's defaults + * @param inputType For search, ingest etc * @param listener Inference result listener */ void infer( @@ -86,6 +87,27 @@ void infer( ActionListener listener ); + /** + * Chunk long text according to {@code chunkingOptions} or the + * model defaults if {@code chunkingOptions} contains unset + * values. + * + * @param model The model + * @param input Inference input + * @param taskSettings Settings in the request to override the model's defaults + * @param inputType For search, ingest etc + * @param chunkingOptions The window and span options to apply + * @param listener Inference result listener + */ + void chunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ); + /** * Start or prepare the model for use. * @param model The model diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java index d5973807d9d78..ce6f1b21b734c 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java @@ -13,49 +13,41 @@ import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -public class InferenceServiceRegistry implements Closeable { - - private final Map services; - private final List namedWriteables = new ArrayList<>(); - - public InferenceServiceRegistry( - List inferenceServicePlugins, - InferenceServiceExtension.InferenceServiceFactoryContext factoryContext - ) { - // TODO check names are unique - services = inferenceServicePlugins.stream() - .flatMap(r -> r.getInferenceServiceFactories().stream()) - .map(factory -> factory.create(factoryContext)) - .collect(Collectors.toMap(InferenceService::name, Function.identity())); - } - public void init(Client client) { - services.values().forEach(s -> s.init(client)); - } +public interface InferenceServiceRegistry extends Closeable { + void init(Client client); - public Map getServices() { - return services; - } + Map getServices(); - public Optional getService(String serviceName) { - return Optional.ofNullable(services.get(serviceName)); - } + Optional getService(String serviceName); - public List getNamedWriteables() { - return namedWriteables; - } + List getNamedWriteables(); + + class NoopInferenceServiceRegistry implements InferenceServiceRegistry { + public NoopInferenceServiceRegistry() {} - @Override - public void close() throws IOException { - for (var service : services.values()) { - service.close(); + @Override + public void init(Client client) {} + + @Override + public Map getServices() { + return Map.of(); + } + + @Override + public Optional getService(String serviceName) { + return Optional.empty(); } + + @Override + public List getNamedWriteables() { + return List.of(); + } + + @Override + public void close() throws IOException {} } } diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistryImpl.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistryImpl.java new file mode 100644 index 0000000000000..f0a990ded98ce --- /dev/null +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistryImpl.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.inference; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class InferenceServiceRegistryImpl implements InferenceServiceRegistry { + + private final Map services; + private final List namedWriteables = new ArrayList<>(); + + public InferenceServiceRegistryImpl( + List inferenceServicePlugins, + InferenceServiceExtension.InferenceServiceFactoryContext factoryContext + ) { + // TODO check names are unique + services = inferenceServicePlugins.stream() + .flatMap(r -> r.getInferenceServiceFactories().stream()) + .map(factory -> factory.create(factoryContext)) + .collect(Collectors.toMap(InferenceService::name, Function.identity())); + } + + @Override + public void init(Client client) { + services.values().forEach(s -> s.init(client)); + } + + @Override + public Map getServices() { + return services; + } + + @Override + public Optional getService(String serviceName) { + return Optional.ofNullable(services.get(serviceName)); + } + + @Override + public List getNamedWriteables() { + return namedWriteables; + } + + @Override + public void close() throws IOException { + for (var service : services.values()) { + service.close(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/inference/ModelRegistry.java b/server/src/main/java/org/elasticsearch/inference/ModelRegistry.java new file mode 100644 index 0000000000000..fa90d5ba6f756 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/inference/ModelRegistry.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.inference; + +import org.elasticsearch.action.ActionListener; + +import java.util.List; +import java.util.Map; + +public interface ModelRegistry { + + /** + * Get a model. + * Secret settings are not included + * @param inferenceEntityId Model to get + * @param listener Model listener + */ + void getModel(String inferenceEntityId, ActionListener listener); + + /** + * Get a model with its secret settings + * @param inferenceEntityId Model to get + * @param listener Model listener + */ + void getModelWithSecrets(String inferenceEntityId, ActionListener listener); + + /** + * Get all models of a particular task type. + * Secret settings are not included + * @param taskType The task type + * @param listener Models listener + */ + void getModelsByTaskType(TaskType taskType, ActionListener> listener); + + /** + * Get all models. + * Secret settings are not included + * @param listener Models listener + */ + void getAllModels(ActionListener> listener); + + void storeModel(Model model, ActionListener listener); + + void deleteModel(String modelId, ActionListener listener); + + /** + * Semi parsed model where inference entity id, task type and service + * are known but the settings are not parsed. + */ + record UnparsedModel( + String inferenceEntityId, + TaskType taskType, + String service, + Map settings, + Map secrets + ) {} + + class NoopModelRegistry implements ModelRegistry { + @Override + public void getModel(String modelId, ActionListener listener) { + fail(listener); + } + + @Override + public void getModelsByTaskType(TaskType taskType, ActionListener> listener) { + listener.onResponse(List.of()); + } + + @Override + public void getAllModels(ActionListener> listener) { + listener.onResponse(List.of()); + } + + @Override + public void storeModel(Model model, ActionListener listener) { + fail(listener); + } + + @Override + public void deleteModel(String modelId, ActionListener listener) { + fail(listener); + } + + @Override + public void getModelWithSecrets(String inferenceEntityId, ActionListener listener) { + fail(listener); + } + + private static void fail(ActionListener listener) { + listener.onFailure(new IllegalArgumentException("No model registry configured")); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 3a2a810dc61b5..1f82ebd786e98 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -41,6 +41,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; @@ -668,10 +669,41 @@ void validatePipeline(Map ingestInfos, String pipelin ExceptionsHelper.rethrowAndSuppress(exceptions); } + private record IngestPipelinesExecutionResult(boolean success, boolean shouldKeep, Exception exception, String failedIndex) { + + private static final IngestPipelinesExecutionResult SUCCESSFUL_RESULT = new IngestPipelinesExecutionResult(true, true, null, null); + private static final IngestPipelinesExecutionResult DISCARD_RESULT = new IngestPipelinesExecutionResult(true, false, null, null); + private static IngestPipelinesExecutionResult failAndStoreFor(String index, Exception e) { + return new IngestPipelinesExecutionResult(false, true, e, index); + } + } + + /** + * Executes all applicable pipelines for a collection of documents. + * @param numberOfActionRequests The total number of requests to process. + * @param actionRequests The collection of requests to be processed. + * @param onDropped A callback executed when a document is dropped by a pipeline. + * Accepts the slot in the collection of requests that the document occupies. + * @param shouldStoreFailure A predicate executed on each ingest failure to determine if the + * failure should be stored somewhere. + * @param onStoreFailure A callback executed when a document fails ingest but the failure should + * be persisted elsewhere. Accepts the slot in the collection of requests + * that the document occupies, the index name that the request was targeting + * at the time of failure, and the exception that the document encountered. + * @param onFailure A callback executed when a document fails ingestion and does not need to be + * persisted. Accepts the slot in the collection of requests that the document + * occupies, and the exception that the document encountered. + * @param onCompletion A callback executed once all documents have been processed. Accepts the thread + * that ingestion completed on or an exception in the event that the entire operation + * has failed. + * @param executorName Which executor the bulk request should be executed on. + */ public void executeBulkRequest( final int numberOfActionRequests, final Iterable> actionRequests, final IntConsumer onDropped, + final Predicate shouldStoreFailure, + final TriConsumer onStoreFailure, final BiConsumer onFailure, final BiConsumer onCompletion, final String executorName @@ -708,34 +740,45 @@ protected void doRun() { totalMetrics.preIngest(); final int slot = i; final Releasable ref = refs.acquire(); + DocumentParsingObserver documentParsingObserver = documentParsingObserverSupplier.get(); + final IngestDocument ingestDocument = newIngestDocument(indexRequest, documentParsingObserver); + final org.elasticsearch.script.Metadata originalDocumentMetadata = ingestDocument.getMetadata().clone(); // the document listener gives us three-way logic: a document can fail processing (1), or it can // be successfully processed. a successfully processed document can be kept (2) or dropped (3). - final ActionListener documentListener = ActionListener.runAfter(new ActionListener<>() { - @Override - public void onResponse(Boolean kept) { - assert kept != null; - if (kept == false) { - onDropped.accept(slot); + final ActionListener documentListener = ActionListener.runAfter( + new ActionListener<>() { + @Override + public void onResponse(IngestPipelinesExecutionResult result) { + assert result != null; + if (result.success) { + if (result.shouldKeep == false) { + onDropped.accept(slot); + } + } else { + // We were given a failure result in the onResponse method, so we must store the failure + // Recover the original document state, track a failed ingest, and pass it along + updateIndexRequestMetadata(indexRequest, originalDocumentMetadata); + totalMetrics.ingestFailed(); + onStoreFailure.apply(slot, result.failedIndex, result.exception); + } } - } - @Override - public void onFailure(Exception e) { - totalMetrics.ingestFailed(); - onFailure.accept(slot, e); + @Override + public void onFailure(Exception e) { + totalMetrics.ingestFailed(); + onFailure.accept(slot, e); + } + }, + () -> { + // regardless of success or failure, we always stop the ingest "stopwatch" and release the ref to indicate + // that we're finished with this document + final long ingestTimeInNanos = System.nanoTime() - startTimeInNanos; + totalMetrics.postIngest(ingestTimeInNanos); + ref.close(); } - }, () -> { - // regardless of success or failure, we always stop the ingest "stopwatch" and release the ref to indicate - // that we're finished with this document - final long ingestTimeInNanos = System.nanoTime() - startTimeInNanos; - totalMetrics.postIngest(ingestTimeInNanos); - ref.close(); - }); - DocumentParsingObserver documentParsingObserver = documentParsingObserverSupplier.get(); - - IngestDocument ingestDocument = newIngestDocument(indexRequest, documentParsingObserver); + ); - executePipelines(pipelines, indexRequest, ingestDocument, documentListener); + executePipelines(pipelines, indexRequest, ingestDocument, shouldStoreFailure, documentListener); indexRequest.setPipelinesHaveRun(); assert actionRequest.index() != null; @@ -825,7 +868,8 @@ private void executePipelines( final PipelineIterator pipelines, final IndexRequest indexRequest, final IngestDocument ingestDocument, - final ActionListener listener + final Predicate shouldStoreFailure, + final ActionListener listener ) { assert pipelines.hasNext(); PipelineSlot slot = pipelines.next(); @@ -835,13 +879,20 @@ private void executePipelines( // reset the reroute flag, at the start of a new pipeline execution this document hasn't been rerouted yet ingestDocument.resetReroute(); + final String originalIndex = indexRequest.indices()[0]; + final Consumer exceptionHandler = (Exception e) -> { + if (shouldStoreFailure.test(originalIndex)) { + listener.onResponse(IngestPipelinesExecutionResult.failAndStoreFor(originalIndex, e)); + } else { + listener.onFailure(e); + } + }; try { if (pipeline == null) { throw new IllegalArgumentException("pipeline with id [" + pipelineId + "] does not exist"); } indexRequest.addPipeline(pipelineId); - final String originalIndex = indexRequest.indices()[0]; executePipeline(ingestDocument, pipeline, (keep, e) -> { assert keep != null; @@ -855,12 +906,12 @@ private void executePipelines( ), e ); - listener.onFailure(e); + exceptionHandler.accept(e); return; // document failed! } if (keep == false) { - listener.onResponse(false); + listener.onResponse(IngestPipelinesExecutionResult.DISCARD_RESULT); return; // document dropped! } @@ -875,7 +926,7 @@ private void executePipelines( } catch (IllegalArgumentException ex) { // An IllegalArgumentException can be thrown when an ingest processor creates a source map that is self-referencing. // In that case, we catch and wrap the exception, so we can include more details - listener.onFailure( + exceptionHandler.accept( new IllegalArgumentException( format( "Failed to generate the source document for ingest pipeline [%s] for document [%s/%s]", @@ -895,7 +946,7 @@ private void executePipelines( if (Objects.equals(originalIndex, newIndex) == false) { // final pipelines cannot change the target index (either directly or by way of a reroute) if (isFinalPipeline) { - listener.onFailure( + exceptionHandler.accept( new IllegalStateException( format( "final pipeline [%s] can't change the target index (from [%s] to [%s]) for document [%s]", @@ -914,7 +965,7 @@ private void executePipelines( if (cycle) { List indexCycle = new ArrayList<>(ingestDocument.getIndexHistory()); indexCycle.add(newIndex); - listener.onFailure( + exceptionHandler.accept( new IllegalStateException( format( "index cycle detected while processing pipeline [%s] for document [%s]: %s", @@ -941,12 +992,12 @@ private void executePipelines( } if (newPipelines.hasNext()) { - executePipelines(newPipelines, indexRequest, ingestDocument, listener); + executePipelines(newPipelines, indexRequest, ingestDocument, shouldStoreFailure, listener); } else { // update the index request's source and (potentially) cache the timestamp for TSDB updateIndexRequestSource(indexRequest, ingestDocument); cacheRawTimestamp(indexRequest, ingestDocument); - listener.onResponse(true); // document succeeded! + listener.onResponse(IngestPipelinesExecutionResult.SUCCESSFUL_RESULT); // document succeeded! } }); } catch (Exception e) { @@ -954,7 +1005,7 @@ private void executePipelines( () -> format("failed to execute pipeline [%s] for document [%s/%s]", pipelineId, indexRequest.index(), indexRequest.id()), e ); - listener.onFailure(e); // document failed! + exceptionHandler.accept(e); // document failed } } diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 0795fef891f91..24c8b87bcff50 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -123,6 +123,8 @@ import org.elasticsearch.indices.recovery.plan.PeerOnlyRecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.ShardSnapshotsService; +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.ModelRegistry; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.monitor.MonitorService; import org.elasticsearch.monitor.fs.FsHealthService; @@ -141,6 +143,7 @@ import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.DiscoveryPlugin; import org.elasticsearch.plugins.HealthPlugin; +import org.elasticsearch.plugins.InferenceRegistryPlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.MetadataUpgrader; @@ -168,6 +171,7 @@ import org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.reservedstate.service.FileSettingsService; +import org.elasticsearch.rest.action.search.SearchResponseMetrics; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; @@ -880,6 +884,7 @@ record PluginServiceInstances( ); final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService); final SearchTransportAPMMetrics searchTransportAPMMetrics = new SearchTransportAPMMetrics(telemetryProvider.getMeterRegistry()); + final SearchResponseMetrics searchResponseMetrics = new SearchResponseMetrics(telemetryProvider.getMeterRegistry()); final SearchTransportService searchTransportService = new SearchTransportService( transportService, client, @@ -1058,6 +1063,7 @@ record PluginServiceInstances( b.bind(MetadataUpdateSettingsService.class).toInstance(metadataUpdateSettingsService); b.bind(SearchService.class).toInstance(searchService); b.bind(SearchTransportAPMMetrics.class).toInstance(searchTransportAPMMetrics); + b.bind(SearchResponseMetrics.class).toInstance(searchResponseMetrics); b.bind(SearchTransportService.class).toInstance(searchTransportService); b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(searchService::aggReduceContextBuilder)); b.bind(Transport.class).toInstance(transport); @@ -1087,6 +1093,18 @@ record PluginServiceInstances( ); } + // Register noop versions of inference services if Inference plugin is not available + Optional inferenceRegistryPlugin = getSinglePlugin(InferenceRegistryPlugin.class); + modules.bindToInstance( + InferenceServiceRegistry.class, + inferenceRegistryPlugin.map(InferenceRegistryPlugin::getInferenceServiceRegistry) + .orElse(new InferenceServiceRegistry.NoopInferenceServiceRegistry()) + ); + modules.bindToInstance( + ModelRegistry.class, + inferenceRegistryPlugin.map(InferenceRegistryPlugin::getModelRegistry).orElse(new ModelRegistry.NoopModelRegistry()) + ); + injector = modules.createInjector(); postInjection(clusterModule, actionModule, clusterService, transportService, featureService); diff --git a/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java index 18e21094fc11d..181471b9b06f4 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestHeaderDefinition; @@ -30,6 +31,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Objects; +import java.util.function.Predicate; import java.util.function.Supplier; /** @@ -70,7 +72,8 @@ default Collection getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Collections.emptyList(); } diff --git a/server/src/main/java/org/elasticsearch/plugins/InferenceRegistryPlugin.java b/server/src/main/java/org/elasticsearch/plugins/InferenceRegistryPlugin.java new file mode 100644 index 0000000000000..696c3a067dad1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/plugins/InferenceRegistryPlugin.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins; + +import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.ModelRegistry; + +/** + * Plugins that provide inference services should implement this interface. + * There should be a single one in the classpath, as we currently support a single instance for ModelRegistry / InfereceServiceRegistry. + */ +public interface InferenceRegistryPlugin { + InferenceServiceRegistry getInferenceServiceRegistry(); + + ModelRegistry getModelRegistry(); +} diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java new file mode 100644 index 0000000000000..00f1f5d5804d6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.rest.action.search; + +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +public class SearchResponseMetrics { + + public static final String TOOK_DURATION_TOTAL_HISTOGRAM_NAME = "es.search_response.took_durations.histogram"; + + private final LongHistogram tookDurationTotalMillisHistogram; + + public SearchResponseMetrics(MeterRegistry meterRegistry) { + this( + meterRegistry.registerLongHistogram( + TOOK_DURATION_TOTAL_HISTOGRAM_NAME, + "The SearchResponse.took durations in milliseconds, expressed as a histogram", + "millis" + ) + ); + } + + private SearchResponseMetrics(LongHistogram tookDurationTotalMillisHistogram) { + this.tookDurationTotalMillisHistogram = tookDurationTotalMillisHistogram; + } + + public long recordTookTime(long tookTime) { + tookDurationTotalMillisHistogram.record(tookTime); + return tookTime; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 773934615e051..7f9e808db9560 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -33,7 +33,9 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.mapper.IdLoader; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -74,11 +76,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.LongSupplier; import java.util.function.ToLongFunction; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; + final class DefaultSearchContext extends SearchContext { private final ReaderContext readerContext; @@ -202,7 +208,8 @@ final class DefaultSearchContext extends SearchContext { searcher, request::nowInMillis, shardTarget.getClusterAlias(), - request.getRuntimeMappings() + request.getRuntimeMappings(), + request.source() == null ? null : request.source().size() ); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; @@ -300,7 +307,7 @@ public void preProcess() { return; } long from = from() == -1 ? 0 : from(); - long size = size() == -1 ? 10 : size(); + long size = size() == -1 ? DEFAULT_SIZE : size(); long resultWindow = from + size; int maxResultWindow = indexService.getIndexSettings().getMaxResultWindow(); @@ -891,7 +898,22 @@ public SourceLoader newSourceLoader() { public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { var indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); - return IdLoader.createTsIdLoader(indexRouting, indexService.getMetadata().getRoutingPaths()); + List routingPaths = indexService.getMetadata().getRoutingPaths(); + for (String routingField : routingPaths) { + if (routingField.contains("*")) { + // In case the routing fields include path matches, find any matches and add them as distinct fields + // to the routing path. + Set matchingRoutingPaths = new TreeSet<>(routingPaths); + for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) { + if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.name())) { + matchingRoutingPaths.add(mapper.name()); + } + } + routingPaths = new ArrayList<>(matchingRoutingPaths); + break; + } + } + return IdLoader.createTsIdLoader(indexRouting, routingPaths); } else { return IdLoader.fromLeafStoredFieldLoader(); } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 7f08089d0c768..51b2e62159a4d 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -11,7 +11,6 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -671,6 +670,8 @@ public double parseDouble(String value, boolean roundUp, LongSupplier now) { * DocValues format for time series id. */ class TimeSeriesIdDocValueFormat implements DocValueFormat { + private static final Base64.Decoder BASE64_DECODER = Base64.getUrlDecoder(); + private TimeSeriesIdDocValueFormat() {} @Override @@ -686,13 +687,41 @@ public String toString() { return "tsid"; } + /** + * @param value The TSID as a {@link BytesRef} + * @return the Base 64 encoded TSID + */ @Override public Object format(BytesRef value) { - return TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(value).streamInput()); + try { + // NOTE: if the tsid is a map of dimension key/value pairs (as it was before introducing + // tsid hashing) we just decode the map and return it. + return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); + } catch (Exception e) { + // NOTE: otherwise the _tsid field is just a hash and we can't decode it + return TimeSeriesIdFieldMapper.encodeTsid(value); + } } @Override public BytesRef parseBytesRef(Object value) { + if (value instanceof BytesRef valueAsBytesRef) { + return valueAsBytesRef; + } + if (value instanceof String valueAsString) { + return new BytesRef(BASE64_DECODER.decode(valueAsString)); + } + return parseBytesRefMap(value); + } + + /** + * After introducing tsid hashing this tsid parsing logic is deprecated. + * Tsid hashing does not allow us to parse the tsid extracting dimension fields key/values pairs. + * @param value The Map encoding tsid dimension fields key/value pairs. + * + * @return a {@link BytesRef} representing a map of key/value pairs + */ + private BytesRef parseBytesRefMap(Object value) { if (value instanceof Map == false) { throw new IllegalArgumentException("Cannot parse tsid object [" + value + "]"); } @@ -718,7 +747,8 @@ public BytesRef parseBytesRef(Object value) { } try { - return builder.build().toBytesRef(); + // NOTE: we can decode the tsid only if it is not hashed (represented as a map) + return builder.buildLegacyTsid().toBytesRef(); } catch (IOException e) { throw new IllegalArgumentException(e); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java index 273df99f6479c..14c0f5289c48f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java @@ -44,7 +44,7 @@ public LeafReaderContext getLeafReaderContext() { return leafReaderContext; } - public BytesRef getTsid() { + public BytesRef getTsidHash() { return tsidProvider != null ? tsidProvider.get() : null; } @@ -52,7 +52,7 @@ public long getTimestamp() { return timestampProvider.getAsLong(); } - public int getTsidOrd() { + public int getTsidHashOrd() { if (tsidOrdProvider == null) { throw new IllegalArgumentException( "Aggregation on a time-series field is misconfigured, likely due to lack of wrapping " diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferableBucketAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferableBucketAggregator.java index 192d3b1f84858..79dcdd6e0b220 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferableBucketAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferableBucketAggregator.java @@ -28,6 +28,7 @@ public abstract class DeferableBucketAggregator extends BucketsAggregator { */ private DeferringBucketCollector deferringCollector; private List deferredAggregationNames; + private final boolean inSortOrderExecutionRequired; protected DeferableBucketAggregator( String name, @@ -38,6 +39,7 @@ protected DeferableBucketAggregator( ) throws IOException { // Assumes that we're collecting MANY buckets. super(name, factories, context, parent, CardinalityUpperBound.MANY, metadata); + this.inSortOrderExecutionRequired = context.isInSortOrderExecutionRequired(); } @Override @@ -46,6 +48,15 @@ protected void doPreCollection() throws IOException { List deferredAggregations = null; for (int i = 0; i < subAggregators.length; ++i) { if (shouldDefer(subAggregators[i])) { + // Deferred collection isn't possible with TimeSeriesIndexSearcher, + // this will always result in incorrect results. The is caused by + // the fact that tsid will not be correctly recorded, because when + // deferred collection occurs the TimeSeriesIndexSearcher already + // completed execution. + if (inSortOrderExecutionRequired) { + throw new IllegalArgumentException("[" + name + "] aggregation is incompatible with time series execution mode"); + } + if (deferringCollector == null) { deferringCollector = buildDeferringCollector(); deferredAggregations = new ArrayList<>(subAggregators.length); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index b078b62c4b82d..7eb4cf68e582b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -247,20 +247,24 @@ protected AggregatorFactory doBuild( Object obj = after.get(sourceName); if (configs[i].missingBucket() && obj == null) { values[i] = null; - } else if (obj instanceof Comparable c) { - values[i] = c; - } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { - // If input is a _tsid map, encode the map to the _tsid BytesRef - values[i] = configs[i].format().parseBytesRef(obj); - } else { - throw new IllegalArgumentException( - "Invalid value for [after." - + sources.get(i).name() - + "], expected comparable, got [" - + (obj == null ? "null" : obj.getClass().getSimpleName()) - + "]" - ); - } + } else if (obj instanceof String s + && configs[i].fieldType() != null + && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { + values[i] = configs[i].format().parseBytesRef(s); + } else if (obj instanceof Comparable c) { + values[i] = c; + } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { + // If input is a _tsid map, encode the map to the _tsid BytesRef + values[i] = configs[i].format().parseBytesRef(obj); + } else { + throw new IllegalArgumentException( + "Invalid value for [after." + + sources.get(i).name() + + "], expected comparable, got [" + + (obj == null ? "null" : obj.getClass().getSimpleName()) + + "]" + ); + } } afterKey = new CompositeKey(values); } else { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java index 1ddf208a2a86e..922baf1f83f83 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java @@ -508,14 +508,37 @@ static Object formatObject(Object obj, DocValueFormat format) { } Object formatted = obj; Object parsed; - if (obj.getClass() == BytesRef.class) { + if (obj.getClass() == BytesRef.class && format == DocValueFormat.TIME_SERIES_ID) { + BytesRef value = (BytesRef) obj; + // NOTE: formatting a tsid returns a Base64 encoding of the tsid BytesRef which we cannot use to get back the original tsid + formatted = format.format(value); + parsed = format.parseBytesRef(value); + // NOTE: we cannot parse the Base64 encoding representation of the tsid and get back the original BytesRef + if (parsed.equals(obj) == false) { + throw new IllegalArgumentException( + "Format [" + + format + + "] created output it couldn't parse for value [" + + obj + + "] " + + "of type [" + + obj.getClass() + + "]. formatted value: [" + + formatted + + "(" + + parsed.getClass() + + ")]" + ); + } + } + if (obj.getClass() == BytesRef.class && format != DocValueFormat.TIME_SERIES_ID) { BytesRef value = (BytesRef) obj; if (format == DocValueFormat.RAW) { formatted = value.utf8ToString(); } else { formatted = format.format(value); } - parsed = format.parseBytesRef(formatted); + parsed = format.parseBytesRef(formatted.toString()); if (parsed.equals(obj) == false) { throw new IllegalArgumentException( "Format [" diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregationBuilder.java similarity index 93% rename from x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregationBuilder.java index 31be7f149831d..4f71c964ebaf9 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregationBuilder.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.countedkeyword; +package org.elasticsearch.search.aggregations.bucket.countedterms; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -56,7 +57,7 @@ public CountedTermsAggregationBuilder(String name) { super(name); } - protected CountedTermsAggregationBuilder( + public CountedTermsAggregationBuilder( ValuesSourceAggregationBuilder clone, AggregatorFactories.Builder factoriesBuilder, Map metadata @@ -64,7 +65,7 @@ protected CountedTermsAggregationBuilder( super(clone, factoriesBuilder, metadata); } - protected CountedTermsAggregationBuilder(StreamInput in) throws IOException { + public CountedTermsAggregationBuilder(StreamInput in) throws IOException { super(in); bucketCountThresholds = new TermsAggregator.BucketCountThresholds(in); } diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java similarity index 96% rename from x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregator.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java index 5e1b1e3624f00..588c53a2d1463 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregator.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.countedkeyword; +package org.elasticsearch.search.aggregations.bucket.countedterms; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregatorFactory.java similarity index 95% rename from x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorFactory.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregatorFactory.java index 3b8be76f14da8..430e28e96d5ee 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregatorFactory.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.countedkeyword; +package org.elasticsearch.search.aggregations.bucket.countedterms; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.rest.RestStatus; diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregatorSupplier.java similarity index 81% rename from x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorSupplier.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregatorSupplier.java index 2817863f6b42c..979c99018e969 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/countedterms/CountedTermsAggregatorSupplier.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.countedkeyword; +package org.elasticsearch.search.aggregations.bucket.countedterms; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index bc4b2a85bab68..6e15b7b0fd012 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -64,6 +64,7 @@ import java.util.Objects; import java.util.function.Consumer; import java.util.function.ToLongFunction; +import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseTopLevelQuery; @@ -1258,6 +1259,7 @@ private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTr parser.getTokenLocation() ); } + List knnBuilders = new ArrayList<>(); SearchUsage searchUsage = new SearchUsage(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -1337,7 +1339,7 @@ private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTr postQueryBuilder = parseTopLevelQuery(parser, searchUsage::trackQueryUsage); searchUsage.trackSectionUsage(POST_FILTER_FIELD.getPreferredName()); } else if (KNN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - knnSearch = List.of(KnnSearchBuilder.fromXContent(parser)); + knnBuilders = List.of(KnnSearchBuilder.fromXContent(parser)); searchUsage.trackSectionUsage(KNN_FIELD.getPreferredName()); } else if (RANK_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { if (RANK_SUPPORTED == false) { @@ -1534,10 +1536,9 @@ private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTr searchAfterBuilder = SearchAfterBuilder.fromXContent(parser); searchUsage.trackSectionUsage(SEARCH_AFTER.getPreferredName()); } else if (KNN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - knnSearch = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { - knnSearch.add(KnnSearchBuilder.fromXContent(parser)); + knnBuilders.add(KnnSearchBuilder.fromXContent(parser)); } else { throw new XContentParseException( parser.getTokenLocation(), @@ -1579,6 +1580,7 @@ private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTr throw new ParsingException(parser.getTokenLocation(), "Unexpected token [" + token + "] found after the main object."); } } + knnSearch = knnBuilders.stream().map(knnBuilder -> knnBuilder.build(size())).collect(Collectors.toList()); searchUsageConsumer.accept(searchUsage); return this; } diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java index dab127e8b4e56..44dddc119925f 100644 --- a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java @@ -181,7 +181,7 @@ private static void executeKnnVectorQuery(SearchContext context) throws IOExcept } SearchExecutionContext searchExecutionContext = context.getSearchExecutionContext(); - List knnSearch = context.request().source().knnSearch(); + List knnSearch = source.knnSearch(); List knnVectorQueryBuilders = knnSearch.stream().map(KnnSearchBuilder::toQueryBuilder).toList(); // Since we apply boost during the DfsQueryPhase, we should not apply boost here: knnVectorQueryBuilders.forEach(knnVectorQueryBuilder -> knnVectorQueryBuilder.boost(DEFAULT_BOOST)); diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 01015ec8cc78e..2368eeb18b021 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -210,7 +210,7 @@ static void addCollectorsAndSearch(SearchContext searchContext) throws QueryPhas if (searcher.timeExceeded()) { assert timeoutRunnable != null : "TimeExceededException thrown even though timeout wasn't set"; if (searchContext.request().allowPartialSearchResults() == false) { - throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded"); + throw new QueryPhaseTimeoutException(searchContext.shardTarget(), "Time exceeded"); } queryResult.searchTimedOut(true); } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhaseTimeoutException.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseTimeoutException.java new file mode 100644 index 0000000000000..1b41f31ea1c82 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseTimeoutException.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.query; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchShardTarget; + +import java.io.IOException; + +/** + * Specific instance of QueryPhaseExecutionException that indicates that a search timeout occurred. + * Always returns http status 504 (Gateway Timeout) + */ +public class QueryPhaseTimeoutException extends QueryPhaseExecutionException { + public QueryPhaseTimeoutException(SearchShardTarget shardTarget, String msg) { + super(shardTarget, msg); + } + + public QueryPhaseTimeoutException(StreamInput in) throws IOException { + super(in); + } + + @Override + public RestStatus status() { + return RestStatus.GATEWAY_TIMEOUT; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java index 347bca245d144..091ce6f8a0f6d 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnByteVectorQuery.java @@ -8,16 +8,35 @@ package org.elasticsearch.search.vectors; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.util.Bits; import org.elasticsearch.search.profile.query.QueryProfiler; +import java.io.IOException; + public class ESKnnByteVectorQuery extends KnnByteVectorQuery implements ProfilingQuery { + private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS; + private long vectorOpsCount; + private final byte[] target; public ESKnnByteVectorQuery(String field, byte[] target, int k, Query filter) { super(field, target, k, filter); + this.target = target; + } + + @Override + protected TopDocs approximateSearch(LeafReaderContext context, Bits acceptDocs, int visitedLimit) throws IOException { + // We increment visit limit by one to bypass a fencepost error in the collector + if (visitedLimit < Integer.MAX_VALUE) { + visitedLimit += 1; + } + TopDocs results = context.reader().searchNearestVectors(field, target, k, acceptDocs, visitedLimit); + return results != null ? results : NO_RESULTS; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java index e83a90a3c4df8..4fa4db1f4ea95 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/ESKnnFloatVectorQuery.java @@ -8,16 +8,24 @@ package org.elasticsearch.search.vectors; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.util.Bits; import org.elasticsearch.search.profile.query.QueryProfiler; +import java.io.IOException; + public class ESKnnFloatVectorQuery extends KnnFloatVectorQuery implements ProfilingQuery { + private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS; private long vectorOpsCount; + private final float[] target; public ESKnnFloatVectorQuery(String field, float[] target, int k, Query filter) { super(field, target, k, filter); + this.target = target; } @Override @@ -27,6 +35,16 @@ protected TopDocs mergeLeafResults(TopDocs[] perLeafResults) { return topK; } + @Override + protected TopDocs approximateSearch(LeafReaderContext context, Bits acceptDocs, int visitedLimit) throws IOException { + // We increment visit limit by one to bypass a fencepost error in the collector + if (visitedLimit < Integer.MAX_VALUE) { + visitedLimit += 1; + } + TopDocs results = context.reader().searchNearestVectors(field, target, k, acceptDocs, visitedLimit); + return results != null ? results : NO_RESULTS; + } + @Override public void profile(QueryProfiler queryProfiler) { queryProfiler.setVectorOpsCount(vectorOpsCount); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java index 318602892daae..4e9c854605f65 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java @@ -34,6 +34,8 @@ import static org.elasticsearch.TransportVersions.V_8_11_X; import static org.elasticsearch.common.Strings.format; +import static org.elasticsearch.index.query.AbstractQueryBuilder.DEFAULT_BOOST; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -41,7 +43,9 @@ * Defines a kNN search to run in the search request. */ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewriteable { - private static final int NUM_CANDS_LIMIT = 10000; + public static final int NUM_CANDS_LIMIT = 10_000; + public static final float NUM_CANDS_MULTIPLICATIVE_FACTOR = 1.5f; + public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); @@ -53,7 +57,7 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("knn", args -> { + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("knn", args -> { // TODO optimize parsing for when BYTE values are provided List vector = (List) args[1]; final float[] vectorArray; @@ -65,21 +69,19 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea } else { vectorArray = null; } - return new KnnSearchBuilder( - (String) args[0], - vectorArray, - (QueryVectorBuilder) args[4], - (int) args[2], - (int) args[3], - (Float) args[5] - ); + return new Builder().field((String) args[0]) + .queryVector(vectorArray) + .queryVectorBuilder((QueryVectorBuilder) args[4]) + .k((Integer) args[2]) + .numCandidates((Integer) args[3]) + .similarity((Float) args[5]); }); static { PARSER.declareString(constructorArg(), FIELD_FIELD); PARSER.declareFloatArray(optionalConstructorArg(), QUERY_VECTOR_FIELD); - PARSER.declareInt(constructorArg(), K_FIELD); - PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); + PARSER.declareInt(optionalConstructorArg(), K_FIELD); + PARSER.declareInt(optionalConstructorArg(), NUM_CANDS_FIELD); PARSER.declareNamedObject( optionalConstructorArg(), (p, c, n) -> p.namedObject(QueryVectorBuilder.class, n, c), @@ -87,21 +89,21 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea ); PARSER.declareFloat(optionalConstructorArg(), VECTOR_SIMILARITY); PARSER.declareFieldArray( - KnnSearchBuilder::addFilterQueries, + KnnSearchBuilder.Builder::addFilterQueries, (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p), FILTER_FIELD, ObjectParser.ValueType.OBJECT_ARRAY ); - PARSER.declareFloat(KnnSearchBuilder::boost, BOOST_FIELD); + PARSER.declareFloat(KnnSearchBuilder.Builder::boost, BOOST_FIELD); PARSER.declareField( - KnnSearchBuilder::innerHit, + KnnSearchBuilder.Builder::innerHit, (p, c) -> InnerHitBuilder.fromXContent(p), INNER_HITS_FIELD, ObjectParser.ValueType.OBJECT ); } - public static KnnSearchBuilder fromXContent(XContentParser parser) throws IOException { + public static KnnSearchBuilder.Builder fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } @@ -113,7 +115,7 @@ public static KnnSearchBuilder fromXContent(XContentParser parser) throws IOExce final int numCands; final Float similarity; final List filterQueries; - float boost = AbstractQueryBuilder.DEFAULT_BOOST; + float boost = DEFAULT_BOOST; InnerHitBuilder innerHitBuilder; /** @@ -130,6 +132,7 @@ public KnnSearchBuilder(String field, float[] queryVector, int k, int numCands, /** * Defines a kNN search where the query vector will be provided by the queryVectorBuilder + * * @param field the name of the vector field to search against * @param queryVectorBuilder the query vector builder * @param k the final number of nearest neighbors to return as top hits @@ -153,16 +156,48 @@ private KnnSearchBuilder( int k, int numCands, Float similarity + ) { + this(field, queryVectorBuilder, queryVector, new ArrayList<>(), k, numCands, similarity, null, DEFAULT_BOOST); + } + + private KnnSearchBuilder( + String field, + Supplier querySupplier, + Integer k, + Integer numCands, + List filterQueries, + Float similarity + ) { + this.field = field; + this.queryVector = new float[0]; + this.queryVectorBuilder = null; + this.k = k; + this.numCands = numCands; + this.filterQueries = filterQueries; + this.querySupplier = querySupplier; + this.similarity = similarity; + } + + private KnnSearchBuilder( + String field, + QueryVectorBuilder queryVectorBuilder, + float[] queryVector, + List filterQueries, + int k, + int numCandidates, + Float similarity, + InnerHitBuilder innerHitBuilder, + float boost ) { if (k < 1) { throw new IllegalArgumentException("[" + K_FIELD.getPreferredName() + "] must be greater than 0"); } - if (numCands < k) { + if (numCandidates < k) { throw new IllegalArgumentException( "[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot be less than " + "[" + K_FIELD.getPreferredName() + "]" ); } - if (numCands > NUM_CANDS_LIMIT) { + if (numCandidates > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } if (queryVector == null && queryVectorBuilder == null) { @@ -187,28 +222,12 @@ private KnnSearchBuilder( this.queryVector = queryVector == null ? new float[0] : queryVector; this.queryVectorBuilder = queryVectorBuilder; this.k = k; - this.numCands = numCands; - this.filterQueries = new ArrayList<>(); - this.querySupplier = null; + this.numCands = numCandidates; + this.innerHitBuilder = innerHitBuilder; this.similarity = similarity; - } - - private KnnSearchBuilder( - String field, - Supplier querySupplier, - int k, - int numCands, - List filterQueries, - Float similarity - ) { - this.field = field; - this.queryVector = new float[0]; - this.queryVectorBuilder = null; - this.k = k; - this.numCands = numCands; + this.boost = boost; this.filterQueries = filterQueries; - this.querySupplier = querySupplier; - this.similarity = similarity; + this.querySupplier = null; } public KnnSearchBuilder(StreamInput in) throws IOException { @@ -373,9 +392,10 @@ public int hashCode() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(FIELD_FIELD.getPreferredName(), field) - .field(K_FIELD.getPreferredName(), k) - .field(NUM_CANDS_FIELD.getPreferredName(), numCands); + builder.field(FIELD_FIELD.getPreferredName(), field); + builder.field(K_FIELD.getPreferredName(), k); + builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); + if (queryVectorBuilder != null) { builder.startObject(QUERY_VECTOR_BUILDER_FIELD.getPreferredName()); builder.field(queryVectorBuilder.getWriteableName(), queryVectorBuilder); @@ -399,7 +419,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(INNER_HITS_FIELD.getPreferredName(), innerHitBuilder, params); } - if (boost != AbstractQueryBuilder.DEFAULT_BOOST) { + if (boost != DEFAULT_BOOST) { builder.field(BOOST_FIELD.getPreferredName(), boost); } @@ -436,4 +456,82 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(innerHitBuilder); } } + + public static class Builder { + + private String field; + private float[] queryVector; + private QueryVectorBuilder queryVectorBuilder; + private Integer k; + private Integer numCandidates; + private Float similarity; + private final List filterQueries = new ArrayList<>(); + private float boost = DEFAULT_BOOST; + private InnerHitBuilder innerHitBuilder; + + public Builder addFilterQueries(List filterQueries) { + Objects.requireNonNull(filterQueries); + this.filterQueries.addAll(filterQueries); + return this; + } + + public Builder field(String field) { + this.field = field; + return this; + } + + public Builder boost(float boost) { + this.boost = boost; + return this; + } + + public Builder innerHit(InnerHitBuilder innerHitBuilder) { + this.innerHitBuilder = innerHitBuilder; + return this; + } + + public Builder queryVector(float[] queryVector) { + this.queryVector = queryVector; + return this; + } + + public Builder queryVectorBuilder(QueryVectorBuilder queryVectorBuilder) { + this.queryVectorBuilder = queryVectorBuilder; + return this; + } + + public Builder k(Integer k) { + this.k = k; + return this; + } + + public Builder numCandidates(Integer numCands) { + this.numCandidates = numCands; + return this; + } + + public Builder similarity(Float similarity) { + this.similarity = similarity; + return this; + } + + public KnnSearchBuilder build(int size) { + int requestSize = size < 0 ? DEFAULT_SIZE : size; + int adjustedK = k == null ? requestSize : k; + int adjustedNumCandidates = numCandidates == null + ? Math.round(Math.min(NUM_CANDS_LIMIT, NUM_CANDS_MULTIPLICATIVE_FACTOR * adjustedK)) + : numCandidates; + return new KnnSearchBuilder( + field, + queryVectorBuilder, + queryVector, + filterQueries, + adjustedK, + adjustedNumCandidates, + similarity, + innerHitBuilder, + boost + ); + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 703cee87daf28..1dc1f97862035 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -40,6 +40,7 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -50,7 +51,9 @@ */ public class KnnVectorQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "knn"; - private static final int NUM_CANDS_LIMIT = 10000; + private static final int NUM_CANDS_LIMIT = 10_000; + private static final float NUM_CANDS_MULTIPLICATIVE_FACTOR = 1.5f; + public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); public static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); @@ -69,14 +72,13 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder filterQueries = new ArrayList<>(); private final Float vectorSimilarity; - public KnnVectorQueryBuilder(String fieldName, float[] queryVector, int numCands, Float vectorSimilarity) { - if (numCands > NUM_CANDS_LIMIT) { + public KnnVectorQueryBuilder(String fieldName, float[] queryVector, Integer numCands, Float vectorSimilarity) { + if (numCands != null && numCands > NUM_CANDS_LIMIT) { throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); } if (queryVector == null) { @@ -113,7 +115,11 @@ public KnnVectorQueryBuilder(String fieldName, float[] queryVector, int numCands public KnnVectorQueryBuilder(StreamInput in) throws IOException { super(in); this.fieldName = in.readString(); - this.numCands = in.readVInt(); + if (in.getTransportVersion().onOrAfter(TransportVersions.KNN_QUERY_NUMCANDS_AS_OPTIONAL_PARAM)) { + this.numCands = in.readOptionalVInt(); + } else { + this.numCands = in.readVInt(); + } if (in.getTransportVersion().before(TransportVersions.V_8_7_0) || in.getTransportVersion().onOrAfter(TransportVersions.KNN_AS_QUERY_ADDED)) { this.queryVector = in.readFloatArray(); @@ -146,7 +152,7 @@ public Float getVectorSimilarity() { return vectorSimilarity; } - public int numCands() { + public Integer numCands() { return numCands; } @@ -169,8 +175,21 @@ public KnnVectorQueryBuilder addFilterQueries(List filterQueries) @Override protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); - out.writeVInt(numCands); - + if (out.getTransportVersion().onOrAfter(TransportVersions.KNN_QUERY_NUMCANDS_AS_OPTIONAL_PARAM)) { + out.writeOptionalVInt(numCands); + } else { + if (numCands == null) { + throw new IllegalArgumentException( + "[" + + NUM_CANDS_FIELD.getPreferredName() + + "] field was mandatory in previous releases " + + "and is required to be non-null by some nodes. " + + "Please make sure to provide the parameter as part of the request." + ); + } else { + out.writeVInt(numCands); + } + } if (out.getTransportVersion().before(TransportVersions.V_8_7_0) || out.getTransportVersion().onOrAfter(TransportVersions.KNN_AS_QUERY_ADDED)) { out.writeFloatArray(queryVector); @@ -192,7 +211,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.startObject(NAME); builder.field(FIELD_FIELD.getPreferredName(), fieldName); builder.field(QUERY_VECTOR_FIELD.getPreferredName(), queryVector); - builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); + if (numCands != null) { + builder.field(NUM_CANDS_FIELD.getPreferredName(), numCands); + } if (vectorSimilarity != null) { builder.field(VECTOR_SIMILARITY_FIELD.getPreferredName(), vectorSimilarity); } @@ -240,6 +261,10 @@ protected QueryBuilder doRewrite(QueryRewriteContext ctx) throws IOException { @Override protected Query doToQuery(SearchExecutionContext context) throws IOException { MappedFieldType fieldType = context.getFieldType(fieldName); + int requestSize = context.requestSize() == null || context.requestSize() < 0 ? DEFAULT_SIZE : context.requestSize(); + int adjustedNumCands = numCands == null + ? Math.round(Math.min(NUM_CANDS_MULTIPLICATIVE_FACTOR * requestSize, NUM_CANDS_LIMIT)) + : numCands; if (fieldType == null) { throw new IllegalArgumentException("field [" + fieldName + "] does not exist in the mapping"); } @@ -284,9 +309,9 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { if (filterQuery != null) { filterQuery = new ToChildBlockJoinQuery(filterQuery, parentFilter); } - return vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery, vectorSimilarity, parentFilter); + return vectorFieldType.createKnnQuery(queryVector, adjustedNumCands, filterQuery, vectorSimilarity, parentFilter); } - return vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery, vectorSimilarity, null); + return vectorFieldType.createKnnQuery(queryVector, adjustedNumCands, filterQuery, vectorSimilarity, null); } @Override @@ -298,7 +323,7 @@ protected int doHashCode() { protected boolean doEquals(KnnVectorQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Arrays.equals(queryVector, other.queryVector) - && numCands == other.numCands + && Objects.equals(numCands, other.numCands) && Objects.equals(filterQueries, other.filterQueries) && Objects.equals(vectorSimilarity, other.vectorSimilarity); } diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index fef0d93ec86cc..97c9ce755c130 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -96,6 +96,7 @@ public static class Names { public static final String THREAD_POOL_METRIC_NAME_QUEUE = ".threads.queue.size"; public static final String THREAD_POOL_METRIC_NAME_ACTIVE = ".threads.active.current"; public static final String THREAD_POOL_METRIC_NAME_LARGEST = ".threads.largest.current"; + public static final String THREAD_POOL_METRIC_NAME_REJECTED = ".threads.rejected.total"; public enum ThreadPoolType { DIRECT("direct"), @@ -387,7 +388,7 @@ private static ArrayList setupMetrics(MeterRegistry meterRegistry, S ); RejectedExecutionHandler rejectedExecutionHandler = threadPoolExecutor.getRejectedExecutionHandler(); if (rejectedExecutionHandler instanceof EsRejectedExecutionHandler handler) { - handler.registerCounter(meterRegistry, prefix, name); + handler.registerCounter(meterRegistry, prefix + THREAD_POOL_METRIC_NAME_REJECTED, name); } } return instruments; diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat new file mode 100644 index 0000000000000..ff848275f2ba1 --- /dev/null +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -0,0 +1,2 @@ +org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat +org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 3e0d9193ffed9..9d5c47fbccbc6 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -80,6 +80,7 @@ import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.UnsupportedAggregationOnDownsampledIndex; import org.elasticsearch.search.internal.ShardSearchContextId; +import org.elasticsearch.search.query.QueryPhaseTimeoutException; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotException; import org.elasticsearch.snapshots.SnapshotId; @@ -827,6 +828,7 @@ public void testIds() { ids.put(173, TooManyScrollContextsException.class); ids.put(174, AggregationExecutionException.InvalidPath.class); ids.put(175, AutoscalingMissedIndicesUpdateException.class); + ids.put(176, QueryPhaseTimeoutException.class); Map, Integer> reverse = new HashMap<>(); for (Map.Entry> entry : ids.entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java index 5dd85f9ee35d5..289ab715e3e78 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.plugins.ActionPlugin; @@ -46,6 +47,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -155,7 +157,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return singletonList(new RestNodesInfoAction(new SettingsFilter(emptyList())) { @@ -217,7 +220,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return singletonList(new FakeHandler()); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestBuilderTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestBuilderTests.java index 8843801e528a3..27b1104163d67 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestBuilderTests.java @@ -10,8 +10,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; public class BulkRequestBuilderTests extends ESTestCase { @@ -21,17 +19,5 @@ public void testValidation() { bulkRequestBuilder.add(new IndexRequestBuilder(null, randomAlphaOfLength(10))); bulkRequestBuilder.add(new IndexRequest()); expectThrows(IllegalStateException.class, bulkRequestBuilder::request); - - bulkRequestBuilder = new BulkRequestBuilder(null, null); - bulkRequestBuilder.add(new IndexRequestBuilder(null, randomAlphaOfLength(10))); - bulkRequestBuilder.setTimeout(randomTimeValue()); - bulkRequestBuilder.setTimeout(TimeValue.timeValueSeconds(randomIntBetween(1, 30))); - expectThrows(IllegalStateException.class, bulkRequestBuilder::request); - - bulkRequestBuilder = new BulkRequestBuilder(null, null); - bulkRequestBuilder.add(new IndexRequestBuilder(null, randomAlphaOfLength(10))); - bulkRequestBuilder.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values()).getValue()); - bulkRequestBuilder.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); - expectThrows(IllegalStateException.class, bulkRequestBuilder::request); } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java index 5cd1fde9edd9b..763dd87f76db3 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java @@ -38,7 +38,7 @@ public void testBulkRequestModifier() { } // wrap the bulk request and fail some of the item requests at random - TransportBulkAction.BulkRequestModifier modifier = new TransportBulkAction.BulkRequestModifier(bulkRequest); + BulkRequestModifier modifier = new BulkRequestModifier(bulkRequest); Set failedSlots = new HashSet<>(); for (int i = 0; modifier.hasNext(); i++) { modifier.next(); @@ -91,7 +91,7 @@ public void testPipelineFailures() { originalBulkRequest.add(new IndexRequest("index").id(String.valueOf(i))); } - TransportBulkAction.BulkRequestModifier modifier = new TransportBulkAction.BulkRequestModifier(originalBulkRequest); + BulkRequestModifier modifier = new BulkRequestModifier(originalBulkRequest); final List failures = new ArrayList<>(); // iterate the requests in order, recording that half of them should be failures @@ -147,7 +147,7 @@ public void testNoFailures() { originalBulkRequest.add(new IndexRequest("index").id(String.valueOf(i))); } - TransportBulkAction.BulkRequestModifier modifier = new TransportBulkAction.BulkRequestModifier(originalBulkRequest); + BulkRequestModifier modifier = new BulkRequestModifier(originalBulkRequest); while (modifier.hasNext()) { modifier.next(); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentTests.java b/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentTests.java new file mode 100644 index 0000000000000..92fa67e9a6ffc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.xcontent.ObjectPath; +import org.elasticsearch.xcontent.json.JsonXContent; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.CoreMatchers.startsWith; + +public class FailureStoreDocumentTests extends ESTestCase { + + public void testFailureStoreDocumentConverstion() throws Exception { + IndexRequest source = new IndexRequest("original_index").routing("fake_routing") + .id("1") + .source(JsonXContent.contentBuilder().startObject().field("key", "value").endObject()); + + // The exception will be wrapped for the test to make sure the converter correctly unwraps it + Exception exception = new ElasticsearchException("Test exception please ignore"); + exception = new RemoteTransportException("Test exception wrapper, please ignore", exception); + + String targetIndexName = "rerouted_index"; + long testTime = 1702357200000L; // 2023-12-12T05:00:00.000Z + + IndexRequest convertedRequest = FailureStoreDocument.transformFailedRequest(source, exception, targetIndexName, () -> testTime); + + // Retargeting write + assertThat(convertedRequest.id(), is(nullValue())); + assertThat(convertedRequest.routing(), is(nullValue())); + assertThat(convertedRequest.index(), is(equalTo(targetIndexName))); + assertThat(convertedRequest.opType(), is(DocWriteRequest.OpType.CREATE)); + + // Original document content is no longer in same place + assertThat("Expected original document to be modified", convertedRequest.sourceAsMap().get("key"), is(nullValue())); + + // Assert document contents + assertThat(ObjectPath.eval("@timestamp", convertedRequest.sourceAsMap()), is(equalTo("2023-12-12T05:00:00.000Z"))); + + assertThat(ObjectPath.eval("document.id", convertedRequest.sourceAsMap()), is(equalTo("1"))); + assertThat(ObjectPath.eval("document.routing", convertedRequest.sourceAsMap()), is(equalTo("fake_routing"))); + assertThat(ObjectPath.eval("document.index", convertedRequest.sourceAsMap()), is(equalTo("original_index"))); + assertThat(ObjectPath.eval("document.source.key", convertedRequest.sourceAsMap()), is(equalTo("value"))); + + assertThat(ObjectPath.eval("error.type", convertedRequest.sourceAsMap()), is(equalTo("exception"))); + assertThat(ObjectPath.eval("error.message", convertedRequest.sourceAsMap()), is(equalTo("Test exception please ignore"))); + assertThat( + ObjectPath.eval("error.stack_trace", convertedRequest.sourceAsMap()), + startsWith( + "org.elasticsearch.ElasticsearchException: Test exception please ignore\n" + + "\tat org.elasticsearch.action.bulk.FailureStoreDocumentTests.testFailureStoreDocumentConverstion" + ) + ); + + assertThat(convertedRequest.isWriteToFailureStore(), is(true)); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 188adf396435f..564cf74697194 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.Nullable; @@ -54,13 +55,16 @@ import org.mockito.Captor; import org.mockito.MockitoAnnotations; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; +import java.util.function.Predicate; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.sameInstance; @@ -82,6 +86,7 @@ public class TransportBulkActionIngestTests extends ESTestCase { */ private static final String WITH_DEFAULT_PIPELINE = "index_with_default_pipeline"; private static final String WITH_DEFAULT_PIPELINE_ALIAS = "alias_for_index_with_default_pipeline"; + private static final String WITH_FAILURE_STORE_ENABLED = "data-stream-failure-store-enabled"; private static final Settings SETTINGS = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), true).build(); @@ -95,6 +100,10 @@ public class TransportBulkActionIngestTests extends ESTestCase { /** Arguments to callbacks we want to capture, but which require generics, so we must use @Captor */ @Captor + ArgumentCaptor> redirectPredicate; + @Captor + ArgumentCaptor> redirectHandler; + @Captor ArgumentCaptor> failureHandler; @Captor ArgumentCaptor> completionHandler; @@ -174,7 +183,7 @@ class TestSingleItemBulkWriteAction extends TransportSingleItemBulkWriteAction> req = bulkDocsItr.getValue().iterator(); failureHandler.getValue().accept(0, exception); // have an exception for our one index request indexRequest2.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing - completionHandler.getValue().accept(DUMMY_WRITE_THREAD, null); + assertTrue(redirectPredicate.getValue().test(WITH_FAILURE_STORE_ENABLED + "-1")); // ensure redirects on failure store data stream + assertFalse(redirectPredicate.getValue().test(WITH_DEFAULT_PIPELINE)); // no redirects for random existing indices + assertFalse(redirectPredicate.getValue().test("index")); // no redirects for non-existant indices with no templates + redirectHandler.getValue().apply(2, WITH_FAILURE_STORE_ENABLED + "-1", exception); // exception and redirect for request 3 (slot 2) + completionHandler.getValue().accept(DUMMY_WRITE_THREAD, null); // all ingestion completed assertTrue(action.isExecuted); assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one verifyNoMoreInteractions(transportService); @@ -322,6 +350,8 @@ public void testSingleItemBulkActionIngestLocal() throws Exception { eq(1), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.WRITE) @@ -368,6 +398,8 @@ public void testIngestSystemLocal() throws Exception { eq(bulkRequest.numberOfActions()), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.SYSTEM_WRITE) @@ -401,7 +433,7 @@ public void testIngestForward() throws Exception { ActionTestUtils.execute(action, null, bulkRequest, listener); // should not have executed ingest locally - verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any()); + verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any(), any(), any()); // but instead should have sent to a remote node with the transport service ArgumentCaptor node = ArgumentCaptor.forClass(DiscoveryNode.class); verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture()); @@ -441,7 +473,7 @@ public void testSingleItemBulkActionIngestForward() throws Exception { ActionTestUtils.execute(singleItemBulkWriteAction, null, indexRequest, listener); // should not have executed ingest locally - verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any()); + verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any(), any(), any()); // but instead should have sent to a remote node with the transport service ArgumentCaptor node = ArgumentCaptor.forClass(DiscoveryNode.class); verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture()); @@ -525,6 +557,8 @@ private void validatePipelineWithBulkUpsert(@Nullable String indexRequestIndexNa eq(bulkRequest.numberOfActions()), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.WRITE) @@ -573,6 +607,8 @@ public void testDoExecuteCalledTwiceCorrectly() throws Exception { eq(1), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.WRITE) @@ -667,6 +703,8 @@ public void testFindDefaultPipelineFromTemplateMatch() { eq(1), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.WRITE) @@ -705,6 +743,8 @@ public void testFindDefaultPipelineFromV2TemplateMatch() { eq(1), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.WRITE) @@ -732,6 +772,8 @@ public void testIngestCallbackExceptionHandled() throws Exception { eq(bulkRequest.numberOfActions()), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.WRITE) @@ -769,6 +811,8 @@ private void validateDefaultPipeline(IndexRequest indexRequest) { eq(1), bulkDocsItr.capture(), any(), + any(), + any(), failureHandler.capture(), completionHandler.capture(), eq(Names.WRITE) diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index c3a1747902893..6f3767892e7a4 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -21,7 +21,9 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexAbstraction.ConcreteIndex; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -52,6 +54,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.TimeUnit; @@ -61,6 +64,7 @@ import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; public class TransportBulkActionTests extends ESTestCase { @@ -336,6 +340,100 @@ public void testRejectionAfterCreateIndexIsPropagated() throws Exception { } } + public void testResolveFailureStoreFromMetadata() throws Exception { + assumeThat(DataStream.isFailureStoreEnabled(), is(true)); + + String dataStreamWithFailureStore = "test-data-stream-failure-enabled"; + String dataStreamWithoutFailureStore = "test-data-stream-failure-disabled"; + long testTime = randomMillisUpToYear9999(); + + IndexMetadata backingIndex1 = DataStreamTestHelper.createFirstBackingIndex(dataStreamWithFailureStore, testTime).build(); + IndexMetadata backingIndex2 = DataStreamTestHelper.createFirstBackingIndex(dataStreamWithoutFailureStore, testTime).build(); + IndexMetadata failureStoreIndex1 = DataStreamTestHelper.createFirstFailureStore(dataStreamWithFailureStore, testTime).build(); + + Metadata metadata = Metadata.builder() + .dataStreams( + Map.of( + dataStreamWithFailureStore, + DataStreamTestHelper.newInstance( + dataStreamWithFailureStore, + List.of(backingIndex1.getIndex()), + 1L, + Map.of(), + false, + null, + List.of(failureStoreIndex1.getIndex()) + ), + dataStreamWithoutFailureStore, + DataStreamTestHelper.newInstance( + dataStreamWithoutFailureStore, + List.of(backingIndex2.getIndex()), + 1L, + Map.of(), + false, + null, + List.of() + ) + ), + Map.of() + ) + .indices( + Map.of( + backingIndex1.getIndex().getName(), + backingIndex1, + backingIndex2.getIndex().getName(), + backingIndex2, + failureStoreIndex1.getIndex().getName(), + failureStoreIndex1 + ) + ) + .build(); + + // Data stream with failure store should store failures + assertThat(TransportBulkAction.shouldStoreFailure(dataStreamWithFailureStore, metadata, testTime), is(true)); + // Data stream without failure store should not + assertThat(TransportBulkAction.shouldStoreFailure(dataStreamWithoutFailureStore, metadata, testTime), is(false)); + // An index should not be considered for failure storage + assertThat(TransportBulkAction.shouldStoreFailure(backingIndex1.getIndex().getName(), metadata, testTime), is(false)); + // even if that index is itself a failure store + assertThat(TransportBulkAction.shouldStoreFailure(failureStoreIndex1.getIndex().getName(), metadata, testTime), is(false)); + } + + public void testResolveFailureStoreFromTemplate() throws Exception { + assumeThat(DataStream.isFailureStoreEnabled(), is(true)); + + String dsTemplateWithFailureStore = "test-data-stream-failure-enabled"; + String dsTemplateWithoutFailureStore = "test-data-stream-failure-disabled"; + String indexTemplate = "test-index"; + long testTime = randomMillisUpToYear9999(); + + Metadata metadata = Metadata.builder() + .indexTemplates( + Map.of( + dsTemplateWithFailureStore, + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dsTemplateWithFailureStore + "-*")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, true)) + .build(), + dsTemplateWithoutFailureStore, + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dsTemplateWithoutFailureStore + "-*")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, false)) + .build(), + indexTemplate, + ComposableIndexTemplate.builder().indexPatterns(List.of(indexTemplate + "-*")).build() + ) + ) + .build(); + + // Data stream with failure store should store failures + assertThat(TransportBulkAction.shouldStoreFailure(dsTemplateWithFailureStore + "-1", metadata, testTime), is(true)); + // Data stream without failure store should not + assertThat(TransportBulkAction.shouldStoreFailure(dsTemplateWithoutFailureStore + "-1", metadata, testTime), is(false)); + // An index template should not be considered for failure storage + assertThat(TransportBulkAction.shouldStoreFailure(indexTemplate + "-1", metadata, testTime), is(false)); + } + private BulkRequest buildBulkRequest(List indices) { BulkRequest request = new BulkRequest(); for (String index : indices) { diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index e0eed9daa97f6..604d404c2f519 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.search.SearchResponseMetrics; import org.elasticsearch.search.DummyQueryBuilder; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHits; @@ -1735,7 +1736,8 @@ protected void doWriteTo(StreamOutput out) throws IOException { null, null, null, - new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()) + new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()), + new SearchResponseMetrics(TelemetryProvider.NOOP.getMeterRegistry()) ); CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestBuilderTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestBuilderTests.java new file mode 100644 index 0000000000000..b2ab56c73c584 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestBuilderTests.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.update; + +import org.elasticsearch.test.ESTestCase; + +public class UpdateRequestBuilderTests extends ESTestCase { + + public void testValidation() { + UpdateRequestBuilder updateRequestBuilder = new UpdateRequestBuilder(null); + updateRequestBuilder.setFetchSource(randomAlphaOfLength(10), randomAlphaOfLength(10)); + updateRequestBuilder.setFetchSource(true); + expectThrows(IllegalStateException.class, updateRequestBuilder::request); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java new file mode 100644 index 0000000000000..2f9148e80988e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; + +public class ES813FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { + @Override + protected Codec getCodec() { + return new Lucene99Codec() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new ES813FlatVectorFormat(); + } + }; + } + + public void testSearchWithVisitedLimit() { + assumeTrue("requires graph based vector codec", false); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java new file mode 100644 index 0000000000000..07a922efd21a6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; + +public class ES813Int8FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { + @Override + protected Codec getCodec() { + return new Lucene99Codec() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new ES813Int8FlatVectorFormat(); + } + }; + } + + public void testSearchWithVisitedLimit() { + assumeTrue("requires graph based vector codec", false); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java index e1c4043f42963..883fdfb132b80 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -23,12 +22,12 @@ protected Number missingValue() { } @Override - protected List outOfRangeSpecs() { + protected List outOfRangeSpecs() { return List.of( - OutOfRangeSpec.of(NumberType.BYTE, "128", "is out of range for a byte"), - OutOfRangeSpec.of(NumberType.BYTE, "-129", "is out of range for a byte"), - OutOfRangeSpec.of(NumberType.BYTE, 128, "is out of range for a byte"), - OutOfRangeSpec.of(NumberType.BYTE, -129, "is out of range for a byte") + NumberTypeOutOfRangeSpec.of(NumberType.BYTE, "128", "is out of range for a byte"), + NumberTypeOutOfRangeSpec.of(NumberType.BYTE, "-129", "is out of range for a byte"), + NumberTypeOutOfRangeSpec.of(NumberType.BYTE, 128, "is out of range for a byte"), + NumberTypeOutOfRangeSpec.of(NumberType.BYTE, -129, "is out of range for a byte") ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index aab481b545879..7775f21fe1222 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -348,9 +349,13 @@ public void testEmptyDocumentMapper() { public void testTooManyDimensionFields() { int max; Settings settings; + String dimensionErrorSuffix = ""; if (randomBoolean()) { - max = 21; // By default no more than 21 dimensions per document are supported - settings = getIndexSettings(); + max = 1000; // By default no more than 1000 dimensions per document are supported + settings = Settings.builder() + .put(getIndexSettings()) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), max) + .build(); } else { max = between(1, 10000); settings = Settings.builder() @@ -358,6 +363,7 @@ public void testTooManyDimensionFields() { .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), max) .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), max + 1) .build(); + dimensionErrorSuffix = "dimension "; } IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> createMapperService(settings, mapping(b -> { for (int i = 0; i <= max; i++) { @@ -367,7 +373,10 @@ public void testTooManyDimensionFields() { .endObject(); } }))); - assertThat(e.getMessage(), containsString("Limit of total dimension fields [" + max + "] has been exceeded")); + assertThat( + e.getMessage(), + containsString(String.format(Locale.ROOT, "Limit of total %sfields [" + max + "] has been exceeded", dimensionErrorSuffix)) + ); } public void testDeeplyNestedMapping() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java new file mode 100644 index 0000000000000..03716f8ad4497 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.contains; + +public class DocumentParserContextTests extends ESTestCase { + + private TestDocumentParserContext context = new TestDocumentParserContext(); + private final MapperBuilderContext root = MapperBuilderContext.root(false, false); + + public void testDynamicMapperSizeMultipleMappers() { + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); + assertEquals(1, context.getNewFieldsSize()); + context.addDynamicMapper(new TextFieldMapper.Builder("bar", createDefaultIndexAnalyzers()).build(root)); + assertEquals(2, context.getNewFieldsSize()); + context.addDynamicRuntimeField(new TestRuntimeField("runtime1", "keyword")); + assertEquals(3, context.getNewFieldsSize()); + context.addDynamicRuntimeField(new TestRuntimeField("runtime2", "keyword")); + assertEquals(4, context.getNewFieldsSize()); + } + + public void testDynamicMapperSizeSameFieldMultipleRuntimeFields() { + context.addDynamicRuntimeField(new TestRuntimeField("foo", "keyword")); + context.addDynamicRuntimeField(new TestRuntimeField("foo", "keyword")); + assertEquals(context.getNewFieldsSize(), 1); + } + + public void testDynamicMapperSizeSameFieldMultipleMappers() { + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); + assertEquals(1, context.getNewFieldsSize()); + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); + assertEquals(1, context.getNewFieldsSize()); + } + + public void testAddRuntimeFieldWhenLimitIsReachedViaMapper() { + context = new TestDocumentParserContext( + Settings.builder() + .put("index.mapping.total_fields.limit", 1) + .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) + .build() + ); + assertTrue(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); + assertFalse(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); + assertThat(context.getIgnoredFields(), contains("runtime_field")); + } + + public void testAddFieldWhenLimitIsReachedViaRuntimeField() { + context = new TestDocumentParserContext( + Settings.builder() + .put("index.mapping.total_fields.limit", 1) + .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) + .build() + ); + assertTrue(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); + assertFalse(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); + assertThat(context.getIgnoredFields(), contains("keyword_field")); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index a3706b7ddab18..ed2efb4728b8d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2580,7 +2580,12 @@ same name need to be part of the same mappings (hence the same document). If th assertArrayEquals(new String[] { "LongField ", "LongField " }, fieldStrings); // merge without going through toXContent and reparsing, otherwise the potential leaf path issue gets fixed on its own - Mapping newMapping = MapperService.mergeMappings(mapperService.documentMapper(), mapping, MapperService.MergeReason.MAPPING_UPDATE); + Mapping newMapping = MapperService.mergeMappings( + mapperService.documentMapper(), + mapping, + MapperService.MergeReason.MAPPING_UPDATE, + mapperService.getIndexSettings() + ); DocumentMapper newDocMapper = new DocumentMapper( mapperService.documentParser(), newMapping, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java index b6b9dfcfea9ff..a04712bd1b16b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.script.DoubleFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -27,13 +26,29 @@ protected Number missingValue() { } @Override - protected List outOfRangeSpecs() { + protected List outOfRangeSpecs() { return List.of( - OutOfRangeSpec.of(NumberFieldMapper.NumberType.DOUBLE, "1.7976931348623157E309", "[double] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.DOUBLE, "-1.7976931348623157E309", "[double] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.DOUBLE, Double.NaN, "[double] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.DOUBLE, Double.POSITIVE_INFINITY, "[double] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.DOUBLE, Double.NEGATIVE_INFINITY, "[double] supports only finite values") + NumberTypeOutOfRangeSpec.of( + NumberFieldMapper.NumberType.DOUBLE, + "1.7976931348623157E309", + "[double] supports only finite values" + ), + NumberTypeOutOfRangeSpec.of( + NumberFieldMapper.NumberType.DOUBLE, + "-1.7976931348623157E309", + "[double] supports only finite values" + ), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.DOUBLE, Double.NaN, "[double] supports only finite values"), + NumberTypeOutOfRangeSpec.of( + NumberFieldMapper.NumberType.DOUBLE, + Double.POSITIVE_INFINITY, + "[double] supports only finite values" + ), + NumberTypeOutOfRangeSpec.of( + NumberFieldMapper.NumberType.DOUBLE, + Double.NEGATIVE_INFINITY, + "[double] supports only finite values" + ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 0e4945f7faea8..329d8a795732f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.Explicit; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -16,6 +17,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -59,4 +61,35 @@ public XContentParser parser() { assertEquals(fieldname, dynamicMappers.get(0).name()); assertEquals(expectedType, dynamicMappers.get(0).typeName()); } + + public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOException { + String source = "{\"f1\": \"foobar\"}"; + XContentParser parser = createParser(JsonXContent.jsonXContent, source); + SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); + + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); + RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( + new PassThroughObjectMapper.Builder("labels").setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) + ).build(MapperBuilderContext.root(false, false)); + Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); + + DocumentParserContext ctx = new TestDocumentParserContext(MappingLookup.fromMapping(mapping), sourceToParse) { + @Override + public XContentParser parser() { + return parser; + } + }; + ctx.path().add("labels"); + + // position the parser on the value + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); + parser.nextToken(); + assertTrue(parser.currentToken().isValue()); + DynamicFieldsBuilder.DYNAMIC_TRUE.createDynamicFieldFromValue(ctx, "f1"); + List dynamicMappers = ctx.getDynamicMappers(); + assertEquals(1, dynamicMappers.size()); + assertEquals("labels.f1", dynamicMappers.get(0).name()); + assertEquals("keyword", dynamicMappers.get(0).typeName()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 54db5832c2726..10bd6c667c26e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -1391,7 +1391,7 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() { ); assertThat(exception.getRootCause(), instanceOf(MapperParsingException.class)); assertEquals( - "Tried to add nested object [time] to object [__dynamic__test] which does not support subobjects", + "Tried to add subobject [time] to object [__dynamic__test] which does not support subobjects", exception.getRootCause().getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 4a29dce00436a..6df9fd1f35f52 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -159,7 +159,7 @@ public void testFieldAliasWithDifferentNestedScopes() { private static FieldMapper createFieldMapper(String parent, String name) { return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current()).build( - new MapperBuilderContext(parent, false, false) + new MapperBuilderContext(parent) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java index 3798129ccff29..556d4312bedca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -24,13 +23,13 @@ protected Number missingValue() { } @Override - protected List outOfRangeSpecs() { + protected List outOfRangeSpecs() { return List.of( - OutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, "3.4028235E39", "[float] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, "-3.4028235E39", "[float] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, Float.NaN, "[float] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, Float.POSITIVE_INFINITY, "[float] supports only finite values"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, Float.NEGATIVE_INFINITY, "[float] supports only finite values") + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, "3.4028235E39", "[float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, "-3.4028235E39", "[float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, Float.NaN, "[float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, Float.POSITIVE_INFINITY, "[float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.FLOAT, Float.NEGATIVE_INFINITY, "[float] supports only finite values") ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java index cc024efb5f307..8e3cd5d1b3202 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -26,14 +25,14 @@ protected Number missingValue() { } @Override - protected List outOfRangeSpecs() { + protected List outOfRangeSpecs() { return List.of( - OutOfRangeSpec.of(NumberType.HALF_FLOAT, "65520", "[half_float] supports only finite values"), - OutOfRangeSpec.of(NumberType.HALF_FLOAT, "-65520", "[half_float] supports only finite values"), - OutOfRangeSpec.of(NumberType.HALF_FLOAT, "-65520", "[half_float] supports only finite values"), - OutOfRangeSpec.of(NumberType.HALF_FLOAT, Float.NaN, "[half_float] supports only finite values"), - OutOfRangeSpec.of(NumberType.HALF_FLOAT, Float.POSITIVE_INFINITY, "[half_float] supports only finite values"), - OutOfRangeSpec.of(NumberType.HALF_FLOAT, Float.NEGATIVE_INFINITY, "[half_float] supports only finite values") + NumberTypeOutOfRangeSpec.of(NumberType.HALF_FLOAT, "65520", "[half_float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberType.HALF_FLOAT, "-65520", "[half_float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberType.HALF_FLOAT, "-65520", "[half_float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberType.HALF_FLOAT, Float.NaN, "[half_float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberType.HALF_FLOAT, Float.POSITIVE_INFINITY, "[half_float] supports only finite values"), + NumberTypeOutOfRangeSpec.of(NumberType.HALF_FLOAT, Float.NEGATIVE_INFINITY, "[half_float] supports only finite values") ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 6712d1c40b4ee..5945e5c81856f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -62,9 +62,10 @@ public void testSynthesizeIdSimple() throws Exception { LeafReader leafReader = indexReader.leaves().get(0).reader(); assertThat(leafReader.numDocs(), equalTo(3)); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(0)))); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(2)))); + // NOTE: time series data is ordered by (tsid, timestamp) + assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(0)))); + assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(1)))); }; prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); } @@ -234,7 +235,7 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } - BytesRef tsid = builder.build().toBytesRef(); + BytesRef tsid = builder.buildTsidHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); iw.addDocument(fields); } @@ -252,7 +253,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc return TsidExtractingIdFieldMapper.createId( false, routingBuilder, - timeSeriesIdBuilder.build().toBytesRef(), + timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp, new byte[16] ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java index 994b74a25743c..ac29ff52dee43 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -24,12 +23,12 @@ protected Number missingValue() { } @Override - protected List outOfRangeSpecs() { + protected List outOfRangeSpecs() { return List.of( - OutOfRangeSpec.of(NumberType.INTEGER, "2147483648", "is out of range for an integer"), - OutOfRangeSpec.of(NumberType.INTEGER, "-2147483649", "is out of range for an integer"), - OutOfRangeSpec.of(NumberType.INTEGER, 2147483648L, " out of range of int"), - OutOfRangeSpec.of(NumberType.INTEGER, -2147483649L, " out of range of int") + NumberTypeOutOfRangeSpec.of(NumberType.INTEGER, "2147483648", "is out of range for an integer"), + NumberTypeOutOfRangeSpec.of(NumberType.INTEGER, "-2147483649", "is out of range for an integer"), + NumberTypeOutOfRangeSpec.of(NumberType.INTEGER, 2147483648L, " out of range of int"), + NumberTypeOutOfRangeSpec.of(NumberType.INTEGER, -2147483649L, " out of range of int") ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 892dbcb185bdb..546551baf4408 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.tests.analysis.MockLowerCaseFilter; @@ -397,9 +398,16 @@ public void testDimensionExtraLongKeyword() throws IOException { Exception e = expectThrows( DocumentParsingException.class, - () -> mapper.parse(source(b -> b.field("field", randomAlphaOfLengthBetween(1025, 2048)))) + () -> mapper.parse( + source(b -> b.field("field", randomAlphaOfLengthBetween(IndexWriter.MAX_TERM_LENGTH, IndexWriter.MAX_TERM_LENGTH + 100))) + ) + ); + assertThat( + e.getCause().getMessage(), + containsString( + "Document contains at least one immense term in field=\"field\" (whose UTF8 encoding is longer than the max length 32766" + ) ); - assertThat(e.getCause().getMessage(), containsString("Dimension fields must be less than [1024] bytes but was")); } public void testConfigureSimilarity() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java index f2d4431e5c79f..79c89f425c8fe 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -33,14 +32,14 @@ protected Number missingValue() { } @Override - protected List outOfRangeSpecs() { + protected List outOfRangeSpecs() { return List.of( - OutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "9223372036854775808", "out of range for a long"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "1e999999999", "out of range for a long"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "-9223372036854775809", "out of range for a long"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "-1e999999999", "out of range for a long"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, new BigInteger("9223372036854775808"), "out of range of long"), - OutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, new BigInteger("-9223372036854775809"), "out of range of long") + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "9223372036854775808", "out of range for a long"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "1e999999999", "out of range for a long"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "-9223372036854775809", "out of range for a long"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, "-1e999999999", "out of range for a long"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, new BigInteger("9223372036854775808"), "out of range of long"), + NumberTypeOutOfRangeSpec.of(NumberFieldMapper.NumberType.LONG, new BigInteger("-9223372036854775809"), "out of range of long") ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 80c074918b06d..68e7bd6f24664 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -24,9 +24,14 @@ import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; @@ -66,6 +71,7 @@ public void testTotalFieldsLimit() throws Throwable { int totalFieldsLimit = randomIntBetween(1, 10); Settings settings = Settings.builder() .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), totalFieldsLimit) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) .build(); MapperService mapperService = createMapperService( settings, @@ -173,6 +179,7 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { Settings settings = Settings.builder() .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfFieldsIncludingAlias) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) .build(); createMapperService(settings, mapping(b -> { b.startObject("alias").field("type", "alias").field("path", "field").endObject(); @@ -184,6 +191,7 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { int numberOfNonAliasFields = 1; Settings errorSettings = Settings.builder() .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfNonAliasFields) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) .build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> createMapperService(errorSettings, mapping(b -> { b.startObject("alias").field("type", "alias").field("path", "field").endObject(); @@ -1241,4 +1249,222 @@ public void testPropertiesField() throws IOException { assertThat(grandchildMapper, instanceOf(FieldMapper.class)); assertEquals("keyword", grandchildMapper.typeName()); } + + public void testMergeUntilLimit() throws IOException { + CompressedXContent mapping1 = new CompressedXContent(""" + { + "properties": { + "parent.child1": { + "type": "keyword" + } + } + }"""); + + CompressedXContent mapping2 = new CompressedXContent(""" + { + "properties": { + "parent.child2": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping1, MergeReason.MAPPING_AUTO_UPDATE); + mapper = mapperService.merge("_doc", mapping2, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().getMapper("parent.child1")); + assertNull(mapper.mappers().getMapper("parent.child2")); + } + + public void testMergeUntilLimitMixedObjectAndDottedNotation() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "parent": { + "properties": { + "child1": { + "type": "keyword" + } + } + }, + "parent.child2": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + assertEquals(0, mapper.mappers().remainingFieldsUntilLimit(2)); + assertNotNull(mapper.mappers().objectMappers().get("parent")); + // the order is not deterministic, but we expect one to be null and the other to be non-null + assertTrue(mapper.mappers().getMapper("parent.child1") == null ^ mapper.mappers().getMapper("parent.child2") == null); + } + + public void testUpdateMappingWhenAtLimit() throws IOException { + CompressedXContent mapping1 = new CompressedXContent(""" + { + "properties": { + "parent.child1": { + "type": "boolean" + } + } + }"""); + + CompressedXContent mapping2 = new CompressedXContent(""" + { + "properties": { + "parent.child1": { + "type": "boolean", + "ignore_malformed": true + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping1, MergeReason.MAPPING_AUTO_UPDATE); + mapper = mapperService.merge("_doc", mapping2, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().getMapper("parent.child1")); + assertTrue(((BooleanFieldMapper) mapper.mappers().getMapper("parent.child1")).ignoreMalformed()); + } + + public void testMultiFieldsUpdate() throws IOException { + CompressedXContent mapping1 = new CompressedXContent(""" + { + "properties": { + "text_field": { + "type": "text", + "fields": { + "multi_field1": { + "type": "boolean" + } + } + } + } + }"""); + + // changes a mapping parameter for multi_field1 and adds another multi field which is supposed to be ignored + CompressedXContent mapping2 = new CompressedXContent(""" + { + "properties": { + "text_field": { + "type": "text", + "fields": { + "multi_field1": { + "type": "boolean", + "ignore_malformed": true + }, + "multi_field2": { + "type": "keyword" + } + } + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping1, MergeReason.MAPPING_AUTO_UPDATE); + mapper = mapperService.merge("_doc", mapping2, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().getMapper("text_field")); + FieldMapper.MultiFields multiFields = ((TextFieldMapper) mapper.mappers().getMapper("text_field")).multiFields(); + Map multiFieldMap = StreamSupport.stream(multiFields.spliterator(), false) + .collect(Collectors.toMap(FieldMapper::name, Function.identity())); + assertThat(multiFieldMap.keySet(), contains("text_field.multi_field1")); + assertTrue(multiFieldMap.get("text_field.multi_field1").ignoreMalformed()); + } + + public void testMultiFieldExceedsLimit() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "multi_field": { + "type": "text", + "fields": { + "multi_field1": { + "type": "boolean" + } + } + }, + "keyword_field": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + assertNull(mapper.mappers().getMapper("multi_field")); + assertNotNull(mapper.mappers().getMapper("keyword_field")); + } + + public void testMergeUntilLimitInitialMappingExceedsLimit() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "field1": { + "type": "keyword" + }, + "field2": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + // the order is not deterministic, but we expect one to be null and the other to be non-null + assertTrue(mapper.mappers().getMapper("field1") == null ^ mapper.mappers().getMapper("field2") == null); + } + + public void testMergeUntilLimitCapacityOnlyForParent() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "parent.child": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().objectMappers().get("parent")); + assertNull(mapper.mappers().getMapper("parent.child")); + } + } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 8eb824884a591..0737dcb7cb5d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -322,7 +322,7 @@ private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { KeywordFieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); - KeywordFieldMapper fieldMapper = fieldBuilder.build(new MapperBuilderContext("foo.metrics", false, false)); + KeywordFieldMapper fieldMapper = fieldBuilder.build(new MapperBuilderContext("foo.metrics")); assertEquals("host.name", fieldMapper.simpleName()); assertEquals("foo.metrics.host.name", fieldMapper.name()); return new ObjectMapper.Builder("foo", ObjectMapper.Defaults.SUBOBJECTS).add( @@ -332,7 +332,7 @@ private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { private ObjectMapper.Builder createObjectSubobjectsFalseLeafWithMultiField() { TextFieldMapper.Builder fieldBuilder = createTextKeywordMultiField("host.name"); - TextFieldMapper textKeywordMultiField = fieldBuilder.build(new MapperBuilderContext("foo.metrics", false, false)); + TextFieldMapper textKeywordMultiField = fieldBuilder.build(new MapperBuilderContext("foo.metrics")); assertEquals("host.name", textKeywordMultiField.simpleName()); assertEquals("foo.metrics.host.name", textKeywordMultiField.name()); FieldMapper fieldMapper = textKeywordMultiField.multiFields.iterator().next(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 6e958ddbea904..cbb0929b813fc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -407,7 +407,7 @@ public void testSubobjectsFalseWithInnerNested() { b.endObject(); }))); assertEquals( - "Failed to parse mapping: Tried to add nested object [time] to object [service] which does not support subobjects", + "Failed to parse mapping: Tried to add subobject [time] to object [service] which does not support subobjects", exception.getMessage() ); } @@ -457,7 +457,7 @@ public void testSubobjectsFalseRootWithInnerNested() { b.endObject(); }))); assertEquals( - "Failed to parse mapping: Tried to add nested object [metrics.service] to object [_doc] which does not support subobjects", + "Failed to parse mapping: Tried to add subobject [metrics.service] to object [_doc] which does not support subobjects", exception.getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PassThroughObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PassThroughObjectMapperTests.java new file mode 100644 index 0000000000000..40994e2835e2b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/PassThroughObjectMapperTests.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class PassThroughObjectMapperTests extends MapperServiceTestCase { + + public void testSimpleKeyword() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + Mapper mapper = mapperService.mappingLookup().getMapper("labels.dim"); + assertThat(mapper, instanceOf(KeywordFieldMapper.class)); + assertFalse(((KeywordFieldMapper) mapper).fieldType().isDimension()); + } + + public void testKeywordDimension() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("time_series_dimension", "true"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + Mapper mapper = mapperService.mappingLookup().getMapper("labels.dim"); + assertThat(mapper, instanceOf(KeywordFieldMapper.class)); + assertTrue(((KeywordFieldMapper) mapper).fieldType().isDimension()); + } + + public void testDynamic() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("dynamic", "true"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + PassThroughObjectMapper mapper = (PassThroughObjectMapper) mapperService.mappingLookup().objectMappers().get("labels"); + assertEquals(ObjectMapper.Dynamic.TRUE, mapper.dynamic()); + } + + public void testEnabled() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("enabled", "false"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + PassThroughObjectMapper mapper = (PassThroughObjectMapper) mapperService.mappingLookup().objectMappers().get("labels"); + assertEquals(false, mapper.isEnabled()); + } + + public void testSubobjectsThrows() throws IOException { + MapperException exception = expectThrows(MapperException.class, () -> createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough").field("subobjects", "true"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + }))); + + assertEquals( + "Failed to parse mapping: Mapping definition for [labels] has unsupported parameters: [subobjects : true]", + exception.getMessage() + ); + } + + public void testAddSubobjectThrows() throws IOException { + MapperException exception = expectThrows(MapperException.class, () -> createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + { + b.startObject("subobj").field("type", "object"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + }))); + + assertEquals( + "Failed to parse mapping: Tried to add subobject [subobj] to object [labels] which does not support subobjects", + exception.getMessage() + ); + } + + public void testWithoutMappers() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + b.startObject("shallow").field("type", "passthrough"); + b.endObject(); + })); + + var labels = mapperService.mappingLookup().objectMappers().get("labels"); + var shallow = mapperService.mappingLookup().objectMappers().get("shallow"); + assertThat(labels.withoutMappers().toString(), equalTo(shallow.toString().replace("shallow", "labels"))); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index b2a6651142181..662a809e6d065 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -8,9 +8,11 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -18,6 +20,8 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -342,6 +346,175 @@ public void testRuntimeSectionRemainingField() throws IOException { assertEquals("Failed to parse mapping: unknown parameter [unsupported] on runtime field [field] of type [keyword]", e.getMessage()); } + public void testPassThroughObjectWithAliases() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("labels").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + assertThat(mapperService.mappingLookup().getMapper("dim"), instanceOf(FieldAliasMapper.class)); + assertThat(mapperService.mappingLookup().getMapper("labels.dim"), instanceOf(KeywordFieldMapper.class)); + } + + public void testPassThroughObjectNested() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("resource").field("type", "object"); + { + b.startObject("properties"); + { + b.startObject("attributes").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("attributes").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("another.dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + })); + assertThat(mapperService.mappingLookup().getMapper("dim"), instanceOf(FieldAliasMapper.class)); + assertThat(mapperService.mappingLookup().getMapper("resource.attributes.dim"), instanceOf(KeywordFieldMapper.class)); + assertThat(mapperService.mappingLookup().getMapper("another.dim"), instanceOf(FieldAliasMapper.class)); + assertThat(mapperService.mappingLookup().getMapper("attributes.another.dim"), instanceOf(KeywordFieldMapper.class)); + } + + public void testAliasMappersCreatesAlias() throws Exception { + var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new PassThroughObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ) + ), + aliases, + context, + 0 + ); + assertEquals(1, aliases.size()); + assertThat(aliases.get("host"), instanceOf(FieldAliasMapper.class)); + } + + public void testAliasMappersCreatesAliasNested() throws Exception { + var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "outer", + new ObjectMapper( + "outer", + "outer", + Explicit.EXPLICIT_TRUE, + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of( + "inner", + new PassThroughObjectMapper( + "inner", + "outer.inner", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ) + ) + ) + ), + aliases, + context, + 0 + ); + assertEquals(1, aliases.size()); + assertThat(aliases.get("host"), instanceOf(FieldAliasMapper.class)); + } + + public void testAliasMappersExitsInDeepNesting() throws Exception { + var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new PassThroughObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ) + ), + aliases, + context, + 1_000_000 + ); + assertTrue(aliases.isEmpty()); + } + + public void testAliasMappersCreatesNoAliasForRegularObject() throws Exception { + var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new ObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + Explicit.EXPLICIT_FALSE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)) + ) + ), + aliases, + context, + 0 + ); + assertTrue(aliases.isEmpty()); + } + + public void testAliasMappersConflictingField() throws Exception { + var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new PassThroughObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ), + "host", + new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context) + ), + aliases, + context, + 0 + ); + assertTrue(aliases.isEmpty()); + } + public void testEmptyType() throws Exception { String mapping = Strings.toString( XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java index b78cdbb8f2bfb..71cfe8b6bb50a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -24,12 +23,12 @@ protected Number missingValue() { } @Override - protected List outOfRangeSpecs() { + protected List outOfRangeSpecs() { return List.of( - OutOfRangeSpec.of(NumberType.SHORT, "32768", "is out of range for a short"), - OutOfRangeSpec.of(NumberType.SHORT, "-32769", "is out of range for a short"), - OutOfRangeSpec.of(NumberType.SHORT, 32768, "out of range of Java short"), - OutOfRangeSpec.of(NumberType.SHORT, -32769, "out of range of Java short") + NumberTypeOutOfRangeSpec.of(NumberType.SHORT, "32768", "is out of range for a short"), + NumberTypeOutOfRangeSpec.of(NumberType.SHORT, "-32769", "is out of range for a short"), + NumberTypeOutOfRangeSpec.of(NumberType.SHORT, 32768, "out of range of Java short"), + NumberTypeOutOfRangeSpec.of(NumberType.SHORT, -32769, "out of range of Java short") ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 07c224fc561d1..d4dc03d22441b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -15,14 +15,13 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.List; -import java.util.Map; -import static org.elasticsearch.test.MapMatcher.assertMap; -import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -46,6 +45,15 @@ protected void registerParameters(ParameterChecker checker) throws IOException { // There aren't any parameters } + @Override + protected IndexVersion getVersion() { + return IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_8_8_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_ID_HASHING) + ); + } + private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { return createMapperService( getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) @@ -71,6 +79,7 @@ private static BytesRef parseAndGetTsid(DocumentMapper docMapper, CheckedConsume return parseDocument(docMapper, f).rootDoc().getBinaryValue(TimeSeriesIdFieldMapper.NAME); } + @SuppressWarnings("unchecked") public void testEnabledInTimeSeriesMode() throws Exception { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -84,10 +93,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ); assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", "value").entry("b", 100L) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); } public void testDisabledInStandardMode() throws Exception { @@ -114,6 +120,7 @@ public void testIncludeInDocumentNotAllowed() throws Exception { /** * Test with non-randomized string for sanity checking. */ + @SuppressWarnings("unchecked") public void testStrings() throws IOException { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -131,12 +138,10 @@ public void testStrings() throws IOException { docMapper, b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("a", "foo").entry("o.e", "bort") - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWE"); } + @SuppressWarnings("unchecked") public void testUnicodeKeys() throws IOException { String fire = new String(new int[] { 0x1F525 }, 0, 1); String coffee = "\u2615"; @@ -146,37 +151,29 @@ public void testUnicodeKeys() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); - Map tsid = TimeSeriesIdFieldMapper.decodeTsid( - new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) - ); - assertMap(tsid, matchesMap().entry(coffee, "good").entry(fire, "hot")); - // Also make sure the keys are in order - assertThat(List.copyOf(tsid.keySet()), equalTo(List.of(coffee, fire))); + Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + assertEquals(tsid, "A-I"); } + @SuppressWarnings("unchecked") public void testKeywordTooLong() throws IOException { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); })); - Exception e = expectThrows( - DocumentParsingException.class, - () -> parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52)).field("@timestamp", "2021-10-01")) - ); - assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1040].")); + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } + @SuppressWarnings("unchecked") public void testKeywordTooLongUtf8() throws IOException { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); })); String theWordLong = "長い"; - Exception e = expectThrows( - DocumentParsingException.class, - () -> parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200)).field("@timestamp", "2021-10-01")) - ); - assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1200].")); + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } public void testKeywordNull() throws IOException { @@ -193,6 +190,7 @@ public void testKeywordNull() throws IOException { /** * Test with non-randomized longs for sanity checking. */ + @SuppressWarnings("unchecked") public void testLong() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -214,10 +212,7 @@ public void testLong() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", 1234).endObject(); }); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", 1234L) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testLongInvalidString() throws IOException { @@ -247,6 +242,7 @@ public void testLongNull() throws IOException { /** * Test with non-randomized integers for sanity checking. */ + @SuppressWarnings("unchecked") public void testInteger() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -268,10 +264,7 @@ public void testInteger() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); }); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Integer.MIN_VALUE) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testIntegerInvalidString() throws IOException { @@ -305,6 +298,7 @@ public void testIntegerOutOfRange() throws IOException { /** * Test with non-randomized shorts for sanity checking. */ + @SuppressWarnings("unchecked") public void testShort() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -326,10 +320,7 @@ public void testShort() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Short.MIN_VALUE).endObject(); }); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Short.MIN_VALUE) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testShortInvalidString() throws IOException { @@ -363,6 +354,7 @@ public void testShortOutOfRange() throws IOException { /** * Test with non-randomized shorts for sanity checking. */ + @SuppressWarnings("unchecked") public void testByte() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -384,10 +376,7 @@ public void testByte() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); }); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Byte.MIN_VALUE) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testByteInvalidString() throws IOException { @@ -421,6 +410,7 @@ public void testByteOutOfRange() throws IOException { /** * Test with non-randomized ips for sanity checking. */ + @SuppressWarnings("unchecked") public void testIp() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -442,10 +432,7 @@ public void testIp() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", "255.255.255.1").endObject(); }); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("kw", "kw").entry("a", "192.168.0.1").entry("o.e", "255.255.255.1") - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); } public void testIpInvalidString() throws IOException { @@ -463,6 +450,7 @@ public void testIpInvalidString() throws IOException { /** * Tests when the total of the tsid is more than 32k. */ + @SuppressWarnings("unchecked") public void testVeryLarge() throws IOException { DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -472,13 +460,19 @@ public void testVeryLarge() throws IOException { })); String large = "many words ".repeat(80); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> { + ParsedDocument doc = parseDocument(docMapper, b -> { b.field("b", "foo"); for (int i = 0; i < 100; i++) { b.field("d" + i, large); } - })); - assertThat(e.getCause().getMessage(), equalTo("_tsid longer than [32766] bytes [88698].")); + }); + + Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + assertEquals( + tsid, + "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd" + + "29yZHMgbWFueSB3b3JkcyA" + ); } /** @@ -575,6 +569,48 @@ public void testDifferentValues() throws IOException { assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } + public void testSameMetricNamesDifferentValues() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); + b.startObject("m2").field("type", "integer").field("time_series_metric", "counter").endObject(); + })); + + ParsedDocument doc1 = parseDocument( + docMapper, + d -> d.field("a", "value") + .field("b", 10) + .field("m1", randomDoubleBetween(100, 200, true)) + .field("m2", randomIntBetween(100, 200)) + ); + ParsedDocument doc2 = parseDocument( + docMapper, + d -> d.field("a", "value").field("b", 10).field("m1", randomDoubleBetween(10, 20, true)).field("m2", randomIntBetween(10, 20)) + ); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + public void testDifferentMetricNamesSameValues() throws IOException { + DocumentMapper docMapper1 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); + })); + + DocumentMapper docMapper2 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m2").field("type", "double").field("time_series_metric", "gauge").endObject(); + })); + + double metricValue = randomDoubleBetween(10, 20, true); + ParsedDocument doc1 = parseDocument(docMapper1, d -> d.field("a", "value").field("b", 10).field("m1", metricValue)); + ParsedDocument doc2 = parseDocument(docMapper2, d -> d.field("a", "value").field("b", 10).field("m2", metricValue)); + // NOTE: plain tsid (not hashed) does not take metric names/values into account + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + /** * Two documents with the same *values* but different dimension keys will generate * different {@code _tsid}s. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 320057e878f1c..c19c21d54a569 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -79,139 +79,273 @@ public static Iterable params() { */ // Dates - items.add(new TestCase("2022-01-01T01:00:00Z", "XsFI2ezm5OViFixWAAABfhMmioA", "{r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("2022-01-01T01:00:01Z", "XsFI2ezm5OViFixWAAABfhMmjmg", "{r1=cat}", "2022-01-01T01:00:01.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:01Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("1970-01-01T00:00:00Z", "XsFI2ezm5OViFixWAAAAAAAAAAA", "{r1=cat}", "1970-01-01T00:00:00.000Z", b -> { - b.field("@timestamp", "1970-01-01T00:00:00Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("-9998-01-01T00:00:00Z", "XsFI2ezm5OViFixW__6oggRgGAA", "{r1=cat}", "-9998-01-01T00:00:00.000Z", b -> { - b.field("@timestamp", "-9998-01-01T00:00:00Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("9998-01-01T00:00:00Z", "XsFI2ezm5OViFixWAADmaSK9hAA", "{r1=cat}", "9998-01-01T00:00:00.000Z", b -> { - b.field("@timestamp", "9998-01-01T00:00:00Z"); - b.field("r1", "cat"); - })); + items.add( + new TestCase( + "2022-01-01T01:00:00Z", + "XsFI2ajcFfi45iV3AAABfhMmioA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "2022-01-01T01:00:01Z", + "XsFI2ajcFfi45iV3AAABfhMmjmg", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "2022-01-01T01:00:01.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:01Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "1970-01-01T00:00:00Z", + "XsFI2ajcFfi45iV3AAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "1970-01-01T00:00:00.000Z", + b -> { + b.field("@timestamp", "1970-01-01T00:00:00Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "-9998-01-01T00:00:00Z", + "XsFI2ajcFfi45iV3__6oggRgGAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "-9998-01-01T00:00:00.000Z", + b -> { + b.field("@timestamp", "-9998-01-01T00:00:00Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "9998-01-01T00:00:00Z", + "XsFI2ajcFfi45iV3AADmaSK9hAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "9998-01-01T00:00:00.000Z", + b -> { + b.field("@timestamp", "9998-01-01T00:00:00Z"); + b.field("r1", "cat"); + } + ) + ); // routing keywords - items.add(new TestCase("r1", "XsFI2ezm5OViFixWAAABfhMmioA", "{r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", (String) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L1", (Long) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("i1", (Integer) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s1", (Short) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b1", (Byte) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("ip1", (String) null); - })); - items.add(new TestCase("r2", "1y-UzdYi98F0UVRiAAABfhMmioA", "{r2=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r2", "cat"); - })); - items.add(new TestCase("o.r3", "zh4dcftpIU55Ond-AAABfhMmioA", "{o.r3=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.startObject("o").field("r3", "cat").endObject(); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("o.r3", "cat"); - })); + items.add( + new TestCase( + "r1", + "XsFI2ajcFfi45iV3AAABfhMmioA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + } + ).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", (String) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", (Long) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i1", (Integer) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", (Short) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", (Byte) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", (String) null); + }) + ); + items.add( + new TestCase( + "r2", + "1y-UzR0iuE1-sOQpAAABfhMmioA", + "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r2", "cat"); + } + ) + ); + items.add( + new TestCase( + "o.r3", + "zh4dcS1h1gf2J5a8AAABfhMmioA", + "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + } + ).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + }) + ); // non-routing keyword - items.add(new TestCase("k1=dog", "XsFI2dL8sZeQhBgxAAABfhMmioA", "{k1=dog, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", "dog"); - })); - items.add(new TestCase("k1=pumpkin", "XsFI2VlD6_SkSo4MAAABfhMmioA", "{k1=pumpkin, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", "pumpkin"); - })); - items.add(new TestCase("k1=empty string", "XsFI2aBA6UgrxLRqAAABfhMmioA", "{k1=, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", ""); - })); - items.add(new TestCase("k2", "XsFI2W2e5Ycw0o5_AAABfhMmioA", "{k2=dog, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k2", "dog"); - })); - items.add(new TestCase("o.k3", "XsFI2ZAfOI6DMQhFAAABfhMmioA", "{o.k3=dog, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.startObject("o").field("k3", "dog").endObject(); - })); - items.add(new TestCase("o.r3", "zh4dcbFtT1qHtjl8AAABfhMmioA", "{o.k3=dog, o.r3=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.startObject("o"); - { - b.field("r3", "cat"); - b.field("k3", "dog"); - } - b.endObject(); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("o.r3", "cat"); - b.startObject("o").field("k3", "dog").endObject(); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.startObject("o").field("r3", "cat").endObject(); - b.field("o.k3", "dog"); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("o.r3", "cat"); - b.field("o.k3", "dog"); - })); - - // long - items.add(new TestCase("L1=1", "XsFI2eGMFOYjW7LLAAABfhMmioA", "{L1=1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L1", 1); - })); items.add( - new TestCase("L1=min", "XsFI2f9V0yuDfkRWAAABfhMmioA", "{L1=" + Long.MIN_VALUE + ", r1=cat}", "2022-01-01T01:00:00.000Z", b -> { + new TestCase( + "k1=dog", + "XsFI2SrEiVgZlSsYAAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", "dog"); + } + ) + ); + items.add( + new TestCase( + "k1=pumpkin", + "XsFI2W8GX8-0QcFxAAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGo", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", "pumpkin"); + } + ) + ); + items.add( + new TestCase( + "k1=empty string", + "XsFI2cna58i6D-Q6AAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1I", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", ""); + } + ) + ); + items.add( + new TestCase( + "k2", + "XsFI2VqlzAuv-06kAAABfhMmioA", + "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k2", "dog"); + } + ) + ); + items.add( + new TestCase( + "o.k3", + "XsFI2S_VhridAKDUAAABfhMmioA", + "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.startObject("o").field("k3", "dog").endObject(); + } + ) + ); + items.add( + new TestCase( + "o.r3", + "zh4dcUwfL7x__2oPAAABfhMmioA", + "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("k3", "dog"); + } + b.endObject(); + } + ).and(b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L1", Long.MIN_VALUE); + b.field("o.r3", "cat"); + b.startObject("o").field("k3", "dog").endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.k3", "dog"); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.k3", "dog"); }) ); - items.add(new TestCase("L2=1234", "XsFI2S8PYEBSm6QYAAABfhMmioA", "{L2=1234, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L2", 1234); - })); + + // long + items.add( + new TestCase( + "L1=1", + "XsFI2fIe53BtV9PCAAABfhMmioA", + "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLac", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", 1); + } + ) + ); + items.add( + new TestCase( + "L1=min", + "XsFI2Qhu7hy1RoXRAAABfhMmioA", + "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6E", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", Long.MIN_VALUE); + } + ) + ); + items.add( + new TestCase( + "L2=1234", + "XsFI2QTrNu7TTpc-AAABfhMmioA", + "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L2", 1234); + } + ) + ); items.add( new TestCase( "o.L3=max", - "zh4dcaI-57LdG7-cAAABfhMmioA", - "{o.L3=" + Long.MAX_VALUE + ", o.r3=cat}", + "zh4dcWBQI6THHqxoAAABfhMmioA", + "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TE", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00.000Z"); @@ -238,16 +372,24 @@ public static Iterable params() { ); // int - items.add(new TestCase("i1=1", "XsFI2R3LiMZSeUGKAAABfhMmioA", "{i1=1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("i1", 1); - })); + items.add( + new TestCase( + "i1=1", + "XsFI2UMS_RWRoHYjAAABfhMmioA", + "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLac", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i1", 1); + } + ) + ); items.add( new TestCase( "i1=min", - "XsFI2fC7DMEVFaU9AAABfhMmioA", - "{i1=" + Integer.MIN_VALUE + ", r1=cat}", + "XsFI2adlQM5ILoA1AAABfhMmioA", + "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -256,16 +398,24 @@ public static Iterable params() { } ) ); - items.add(new TestCase("i2=1234", "XsFI2ZVte8HK90RJAAABfhMmioA", "{i2=1324, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("i2", 1324); - })); + items.add( + new TestCase( + "i2=1234", + "XsFI2bhxfB6J0kBFAAABfhMmioA", + "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpc", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i2", 1324); + } + ) + ); items.add( new TestCase( "o.i3=max", - "zh4dcQy_QJRCqIx7AAABfhMmioA", - "{o.i3=" + Integer.MAX_VALUE + ", o.r3=cat}", + "zh4dcelxKf19CbfdAAABfhMmioA", + "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -292,28 +442,50 @@ public static Iterable params() { ); // short - items.add(new TestCase("s1=1", "XsFI2axCr11Q93m7AAABfhMmioA", "{r1=cat, s1=1}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s1", 1); - })); items.add( - new TestCase("s1=min", "XsFI2Rbs9Ua9BH1wAAABfhMmioA", "{r1=cat, s1=" + Short.MIN_VALUE + "}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s1", Short.MIN_VALUE); - }) + new TestCase( + "s1=1", + "XsFI2Y_y-8kD_BFeAAABfhMmioA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFoc", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", 1); + } + ) + ); + items.add( + new TestCase( + "s1=min", + "XsFI2WV8VNVnmPVNAAABfhMmioA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdM", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", Short.MIN_VALUE); + } + ) + ); + items.add( + new TestCase( + "s2=1234", + "XsFI2VO8mUr-J5CpAAABfhMmioA", + "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9w", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s2", 1234); + } + ) ); - items.add(new TestCase("s2=1234", "XsFI2SBKaLBqXMBYAAABfhMmioA", "{r1=cat, s2=1234}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s2", 1234); - })); items.add( new TestCase( "o.s3=max", - "zh4dcYIFo98LQWs4AAABfhMmioA", - "{o.r3=cat, o.s3=" + Short.MAX_VALUE + "}", + "zh4dcQKh6K11zWeuAAABfhMmioA", + "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -340,28 +512,50 @@ public static Iterable params() { ); // byte - items.add(new TestCase("b1=1", "XsFI2dDrcWaf3zDPAAABfhMmioA", "{b1=1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b1", 1); - })); items.add( - new TestCase("b1=min", "XsFI2cTzLrNqHtxnAAABfhMmioA", "{b1=" + Byte.MIN_VALUE + ", r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b1", Byte.MIN_VALUE); - }) + new TestCase( + "b1=1", + "XsFI2dKxqgT5JDQfAAABfhMmioA", + "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLac", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", 1); + } + ) + ); + items.add( + new TestCase( + "b1=min", + "XsFI2d_PD--DgUvoAAABfhMmioA", + "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1g", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", Byte.MIN_VALUE); + } + ) + ); + items.add( + new TestCase( + "b2=12", + "XsFI2aqX5QjiuhsEAAABfhMmioA", + "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5Vks", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b2", 12); + } + ) ); - items.add(new TestCase("b2=12", "XsFI2Sb77VB9AswjAAABfhMmioA", "{b2=12, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b2", 12); - })); items.add( new TestCase( "o.s3=max", - "zh4dcfFauKzj6lgxAAABfhMmioA", - "{o.b3=" + Byte.MAX_VALUE + ", o.r3=cat}", + "zh4dccJ4YtN_21XHAAABfhMmioA", + "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briE", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -389,22 +583,34 @@ public static Iterable params() { // ip items.add( - new TestCase("ip1=192.168.0.1", "XsFI2dJ1cyrrjNa2AAABfhMmioA", "{ip1=192.168.0.1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("ip1", "192.168.0.1"); - }).and(b -> { + new TestCase( + "ip1=192.168.0.1", + "XsFI2T5km9raIz_rAAABfhMmioA", + "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocE", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "192.168.0.1"); + } + ).and(b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); b.field("r1", "cat"); b.field("ip1", "::ffff:c0a8:1"); }) ); items.add( - new TestCase("ip1=12.12.45.254", "XsFI2ZUAcRxOwhHKAAABfhMmioA", "{ip1=12.12.45.254, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("ip1", "12.12.45.254"); - }).and(b -> { + new TestCase( + "ip1=12.12.45.254", + "XsFI2QWfEH_e_6wIAAABfhMmioA", + "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8Qw", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "12.12.45.254"); + } + ).and(b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); b.field("r1", "cat"); b.field("ip1", "::ffff:c0c:2dfe"); @@ -413,8 +619,8 @@ public static Iterable params() { items.add( new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", - "XsFI2XTGWAekP_oGAAABfhMmioA", - "{ip2=fe80:cd00:0:cde:1257:0:211e:729c, r1=cat}", + "XsFI2WrrLHr1O4iQAAABfhMmioA", + "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuo", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -426,8 +632,8 @@ public static Iterable params() { items.add( new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", - "zh4dcU_FSGP9GuHjAAABfhMmioA", - "{o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348, o.r3=cat}", + "zh4dca7d-9aKOS1MAAABfhMmioA", + "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmI", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -457,8 +663,8 @@ public static Iterable params() { items.add( new TestCase( "huge", - "WZKJR1Fi00B8Afr-AAABfhMmioA", - "{k1=" + huge + ", k2=" + huge.substring(0, 191) + "...}", + "WZKJR_dECvXBSl3xAAABfhMmioA", + "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -603,11 +809,11 @@ public void testSourceDescription() throws IOException { IndexableField tsid = d.rootDoc().getField(TimeSeriesIdFieldMapper.NAME); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(tsid)), - equalTo("a time series document with dimensions " + testCase.expectedTsid) + equalTo("a time series document with tsid " + testCase.expectedTsid) ); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(tsid, timestamp)), - equalTo("a time series document with dimensions " + testCase.expectedTsid + " at [" + testCase.expectedTimestamp + "]") + equalTo("a time series document with tsid " + testCase.expectedTsid + " at [" + testCase.expectedTimestamp + "]") ); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java index 86408b3b22ed7..9d03cc5b8aa8c 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java @@ -44,7 +44,6 @@ public class ShardSplittingQueryTests extends ESTestCase { public void testSplitOnID() throws IOException { - SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = createIndexWriter(dir); @@ -70,7 +69,6 @@ public void testSplitOnID() throws IOException { } public void testSplitOnRouting() throws IOException { - SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = createIndexWriter(dir); @@ -95,7 +93,6 @@ public void testSplitOnRouting() throws IOException { } public void testSplitOnIdOrRouting() throws IOException { - SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = createIndexWriter(dir); @@ -122,7 +119,6 @@ public void testSplitOnIdOrRouting() throws IOException { } public void testSplitOnRoutingPartitioned() throws IOException { - SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = createIndexWriter(dir); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index d345197d88a23..26aa5b1e0454f 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; +import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; @@ -91,6 +92,7 @@ import java.util.function.Consumer; import java.util.function.IntConsumer; import java.util.function.LongSupplier; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -208,7 +210,16 @@ public void testExecuteIndexPipelineDoesNotExist() { @SuppressWarnings("unchecked") final BiConsumer completionHandler = mock(BiConsumer.class); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); assertTrue(failure.get()); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); @@ -1111,6 +1122,8 @@ public String getType() { bulkRequest.numberOfActions(), bulkRequest.requests(), indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), failureHandler, completionHandler, Names.WRITE @@ -1154,6 +1167,8 @@ public void testExecuteBulkPipelineDoesNotExist() { bulkRequest.numberOfActions(), bulkRequest.requests(), indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), failureHandler, completionHandler, Names.WRITE @@ -1218,6 +1233,8 @@ public void close() { bulkRequest.numberOfActions(), bulkRequest.requests(), indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), failureHandler, completionHandler, Names.WRITE @@ -1247,7 +1264,16 @@ public void testExecuteSuccess() { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final BiConsumer completionHandler = mock(BiConsumer.class); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); } @@ -1280,7 +1306,16 @@ public void testDynamicTemplates() throws Exception { CountDownLatch latch = new CountDownLatch(1); final BiConsumer failureHandler = (v, e) -> { throw new AssertionError("must never fail", e); }; final BiConsumer completionHandler = (t, e) -> latch.countDown(); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); latch.await(); assertThat(indexRequest.getDynamicTemplates(), equalTo(Map.of("foo", "bar", "foo.bar", "baz"))); } @@ -1301,7 +1336,16 @@ public void testExecuteEmptyPipeline() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final BiConsumer completionHandler = mock(BiConsumer.class); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); } @@ -1355,7 +1399,16 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final BiConsumer completionHandler = mock(BiConsumer.class); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); verify(processor).execute(any(), any()); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); @@ -1404,7 +1457,16 @@ public void testExecuteFailure() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final BiConsumer completionHandler = mock(BiConsumer.class); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class)); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); @@ -1453,7 +1515,16 @@ public void testExecuteSuccessWithOnFailure() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final BiConsumer completionHandler = mock(BiConsumer.class); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); verify(failureHandler, never()).accept(eq(0), any(IngestProcessorException.class)); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); } @@ -1496,7 +1567,16 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final BiConsumer completionHandler = mock(BiConsumer.class); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class)); verify(completionHandler, times(1)).accept(Thread.currentThread(), null); @@ -1554,6 +1634,8 @@ public void testBulkRequestExecutionWithFailures() throws Exception { numRequest, bulkRequest.requests(), indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), requestItemErrorHandler, completionHandler, Names.WRITE @@ -1563,6 +1645,184 @@ public void testBulkRequestExecutionWithFailures() throws Exception { verify(completionHandler, times(1)).accept(Thread.currentThread(), null); } + public void testExecuteFailureRedirection() throws Exception { + final CompoundProcessor processor = mockCompoundProcessor(); + IngestService ingestService = createWithProcessors( + Map.of( + "mock", + (factories, tag, description, config) -> processor, + "set", + (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail()) + ) + ); + PutPipelineRequest putRequest1 = new PutPipelineRequest( + "_id1", + new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), + XContentType.JSON + ); + // given that set -> fail() above, it's a failure if a document executes against this pipeline + PutPipelineRequest putRequest2 = new PutPipelineRequest( + "_id2", + new BytesArray("{\"processors\": [{\"set\" : {}}]}"), + XContentType.JSON + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty + ClusterState previousClusterState = clusterState; + clusterState = executePut(putRequest1, clusterState); + clusterState = executePut(putRequest2, clusterState); + ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); + final IndexRequest indexRequest = new IndexRequest("_index").id("_id") + .source(Map.of()) + .setPipeline("_id1") + .setFinalPipeline("_id2"); + doThrow(new RuntimeException()).when(processor) + .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); + final Predicate redirectCheck = (idx) -> indexRequest.index().equals(idx); + @SuppressWarnings("unchecked") + final TriConsumer redirectHandler = mock(TriConsumer.class); + @SuppressWarnings("unchecked") + final BiConsumer failureHandler = mock(BiConsumer.class); + @SuppressWarnings("unchecked") + final BiConsumer completionHandler = mock(BiConsumer.class); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + redirectCheck, + redirectHandler, + failureHandler, + completionHandler, + Names.WRITE + ); + verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); + verify(redirectHandler, times(1)).apply(eq(0), eq(indexRequest.index()), any(RuntimeException.class)); + verifyNoInteractions(failureHandler); + verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + } + + public void testExecuteFailureRedirectionWithNestedOnFailure() throws Exception { + final Processor processor = mock(Processor.class); + when(processor.isAsync()).thenReturn(true); + final Processor onFailureProcessor = mock(Processor.class); + when(onFailureProcessor.isAsync()).thenReturn(true); + final Processor onFailureOnFailureProcessor = mock(Processor.class); + when(onFailureOnFailureProcessor.isAsync()).thenReturn(true); + final List processors = List.of(onFailureProcessor); + final List onFailureProcessors = List.of(onFailureOnFailureProcessor); + final CompoundProcessor compoundProcessor = new CompoundProcessor( + false, + List.of(processor), + List.of(new CompoundProcessor(false, processors, onFailureProcessors)) + ); + IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor)); + PutPipelineRequest putRequest = new PutPipelineRequest( + "_id", + new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), + XContentType.JSON + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty + ClusterState previousClusterState = clusterState; + clusterState = executePut(putRequest, clusterState); + ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); + final IndexRequest indexRequest = new IndexRequest("_index").id("_id") + .source(Map.of()) + .setPipeline("_id") + .setFinalPipeline("_none"); + doThrow(new RuntimeException()).when(onFailureOnFailureProcessor) + .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); + doThrow(new RuntimeException()).when(onFailureProcessor) + .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); + doThrow(new RuntimeException()).when(processor) + .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); + final Predicate redirectPredicate = (idx) -> indexRequest.index().equals(idx); + @SuppressWarnings("unchecked") + final TriConsumer redirectHandler = mock(TriConsumer.class); + @SuppressWarnings("unchecked") + final BiConsumer failureHandler = mock(BiConsumer.class); + @SuppressWarnings("unchecked") + final BiConsumer completionHandler = mock(BiConsumer.class); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + redirectPredicate, + redirectHandler, + failureHandler, + completionHandler, + Names.WRITE + ); + verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any()); + verify(redirectHandler, times(1)).apply(eq(0), eq(indexRequest.index()), any(RuntimeException.class)); + verifyNoInteractions(failureHandler); + verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + } + + public void testBulkRequestExecutionWithRedirectedFailures() throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + String pipelineId = "_id"; + + int numRequest = scaledRandomIntBetween(8, 64); + int numIndexRequests = 0; + for (int i = 0; i < numRequest; i++) { + DocWriteRequest request; + if (randomBoolean()) { + if (randomBoolean()) { + request = new DeleteRequest("_index", "_id"); + } else { + request = new UpdateRequest("_index", "_id"); + } + } else { + IndexRequest indexRequest = new IndexRequest("_index").id("_id").setPipeline(pipelineId).setFinalPipeline("_none"); + indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1"); + request = indexRequest; + numIndexRequests++; + } + bulkRequest.add(request); + } + + CompoundProcessor processor = mock(CompoundProcessor.class); + when(processor.isAsync()).thenReturn(true); + when(processor.getProcessors()).thenReturn(List.of(mock(Processor.class))); + Exception error = new RuntimeException(); + doAnswer(args -> { + @SuppressWarnings("unchecked") + BiConsumer handler = (BiConsumer) args.getArguments()[1]; + handler.accept(null, error); + return null; + }).when(processor).execute(any(), any()); + IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor)); + PutPipelineRequest putRequest = new PutPipelineRequest( + "_id", + new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), + XContentType.JSON + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty + ClusterState previousClusterState = clusterState; + clusterState = executePut(putRequest, clusterState); + ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); + + @SuppressWarnings("unchecked") + TriConsumer requestItemRedirectHandler = mock(TriConsumer.class); + @SuppressWarnings("unchecked") + BiConsumer requestItemErrorHandler = mock(BiConsumer.class); + @SuppressWarnings("unchecked") + final BiConsumer completionHandler = mock(BiConsumer.class); + ingestService.executeBulkRequest( + numRequest, + bulkRequest.requests(), + indexReq -> {}, + (s) -> true, + requestItemRedirectHandler, + requestItemErrorHandler, + completionHandler, + Names.WRITE + ); + + verify(requestItemRedirectHandler, times(numIndexRequests)).apply(anyInt(), anyString(), argThat(e -> e.getCause().equals(error))); + verifyNoInteractions(requestItemErrorHandler); + verify(completionHandler, times(1)).accept(Thread.currentThread(), null); + } + public void testBulkRequestExecution() throws Exception { BulkRequest bulkRequest = new BulkRequest(); String pipelineId = "_id"; @@ -1612,6 +1872,8 @@ public void testBulkRequestExecution() throws Exception { numRequest, bulkRequest.requests(), indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), requestItemErrorHandler, completionHandler, Names.WRITE @@ -1721,7 +1983,16 @@ public String execute() { final IndexRequest indexRequest = new IndexRequest("_index"); indexRequest.setPipeline("_id1").setFinalPipeline("_id2"); indexRequest.source(randomAlphaOfLength(10), randomAlphaOfLength(10)); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + (integer, e) -> {}, + (thread, e) -> {}, + Names.WRITE + ); { final IngestStats ingestStats = ingestService.stats(); @@ -1792,7 +2063,16 @@ public void testStats() throws Exception { final IndexRequest indexRequest = new IndexRequest("_index"); indexRequest.setPipeline("_id1").setFinalPipeline("_none"); indexRequest.source(randomAlphaOfLength(10), randomAlphaOfLength(10)); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); final IngestStats afterFirstRequestStats = ingestService.stats(); assertThat(afterFirstRequestStats.pipelineStats().size(), equalTo(2)); @@ -1809,7 +2089,16 @@ public void testStats() throws Exception { assertProcessorStats(0, afterFirstRequestStats, "_id2", 0, 0, 0); indexRequest.setPipeline("_id2"); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); final IngestStats afterSecondRequestStats = ingestService.stats(); assertThat(afterSecondRequestStats.pipelineStats().size(), equalTo(2)); // total @@ -1831,7 +2120,16 @@ public void testStats() throws Exception { clusterState = executePut(putRequest, clusterState); ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); indexRequest.setPipeline("_id1"); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); final IngestStats afterThirdRequestStats = ingestService.stats(); assertThat(afterThirdRequestStats.pipelineStats().size(), equalTo(2)); // total @@ -1854,7 +2152,16 @@ public void testStats() throws Exception { clusterState = executePut(putRequest, clusterState); ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); indexRequest.setPipeline("_id1"); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + failureHandler, + completionHandler, + Names.WRITE + ); final IngestStats afterForthRequestStats = ingestService.stats(); assertThat(afterForthRequestStats.pipelineStats().size(), equalTo(2)); // total @@ -1946,6 +2253,8 @@ public String getDescription() { bulkRequest.numberOfActions(), bulkRequest.requests(), dropHandler, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), failureHandler, completionHandler, Names.WRITE @@ -2030,7 +2339,16 @@ public void testCBORParsing() throws Exception { .setPipeline("_id") .setFinalPipeline("_none"); - ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE); + ingestService.executeBulkRequest( + 1, + List.of(indexRequest), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + (integer, e) -> {}, + (thread, e) -> {}, + Names.WRITE + ); } assertThat(reference.get(), is(instanceOf(byte[].class))); @@ -2101,7 +2419,16 @@ public void testSetsRawTimestamp() { bulkRequest.add(indexRequest6); bulkRequest.add(indexRequest7); bulkRequest.add(indexRequest8); - ingestService.executeBulkRequest(8, bulkRequest.requests(), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE); + ingestService.executeBulkRequest( + 8, + bulkRequest.requests(), + indexReq -> {}, + (s) -> false, + (slot, targetIndex, e) -> fail("Should not be redirecting failures"), + (integer, e) -> {}, + (thread, e) -> {}, + Names.WRITE + ); assertThat(indexRequest1.getRawTimestamp(), nullValue()); assertThat(indexRequest2.getRawTimestamp(), nullValue()); diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 9aa358123d282..bab67233f0025 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.IdLoader; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.MockFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; @@ -65,9 +66,9 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.UUID; @@ -87,7 +88,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class DefaultSearchContextTests extends ESTestCase { +public class DefaultSearchContextTests extends MapperServiceTestCase { public void testPreProcess() throws Exception { TimeValue timeout = new TimeValue(randomIntBetween(1, 100)); @@ -116,7 +117,7 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); - when(indexService.newSearchExecutionContext(eq(shardId.id()), eq(shardId.id()), any(), any(), nullable(String.class), any())) + when(indexService.newSearchExecutionContext(eq(shardId.id()), eq(shardId.id()), any(), any(), nullable(String.class), any(), any())) .thenReturn(searchExecutionContext); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); @@ -475,6 +476,30 @@ public void testNewIdLoaderWithTsdb() throws Exception { } } + public void testNewIdLoaderWithTsdbAndRoutingPathMatch() throws Exception { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-01T00:00:00.000Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2001-01-01T00:00:00.000Z") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "labels.*") + .build(); + + XContentBuilder mappings = mapping(b -> { + b.startObject("labels").field("type", "object"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.endObject(); + } + b.endObject(); + }); + + try (DefaultSearchContext context = createDefaultSearchContext(settings, mappings)) { + assertThat(context.newIdLoader(), instanceOf(IdLoader.TsIdLoader.class)); + context.indexShard().getThreadPool().shutdown(); + } + } + public void testDetermineMaximumNumberOfSlices() { IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(new ShardId("index", "uuid", 0)); @@ -791,6 +816,10 @@ public void testGetFieldCardinalityRuntimeField() { } private DefaultSearchContext createDefaultSearchContext(Settings providedIndexSettings) throws IOException { + return createDefaultSearchContext(providedIndexSettings, null); + } + + private DefaultSearchContext createDefaultSearchContext(Settings providedIndexSettings, XContentBuilder mappings) throws IOException { TimeValue timeout = new TimeValue(randomIntBetween(1, 100)); ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class); when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT); @@ -813,15 +842,23 @@ private DefaultSearchContext createDefaultSearchContext(Settings providedIndexSe SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(indexService.newSearchExecutionContext(eq(shardId.id()), eq(shardId.id()), any(), any(), nullable(String.class), any())) .thenReturn(searchExecutionContext); - MapperService mapperService = mock(MapperService.class); - when(mapperService.hasNested()).thenReturn(randomBoolean()); - when(indexService.mapperService()).thenReturn(mapperService); IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + IndexSettings indexSettings; + MapperService mapperService; + if (mappings != null) { + mapperService = createMapperService(settings, mappings); + indexSettings = mapperService.getIndexSettings(); + } else { + indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + mapperService = mock(MapperService.class); + when(mapperService.hasNested()).thenReturn(randomBoolean()); + when(indexService.mapperService()).thenReturn(mapperService); + when(mapperService.getIndexSettings()).thenReturn(indexSettings); + } when(indexService.getIndexSettings()).thenReturn(indexSettings); + when(indexService.mapperService()).thenReturn(mapperService); when(indexService.getMetadata()).thenReturn(indexMetadata); - when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(searchExecutionContext.getIndexSettings()).thenReturn(indexSettings); when(searchExecutionContext.indexVersionCreated()).thenReturn(indexSettings.getIndexVersionCreated()); diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index def373a6cc861..8c70b35d27c95 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -26,6 +26,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; @@ -377,9 +378,13 @@ public void testParseTsid() throws IOException { timeSeriesIdBuilder.addString("string", randomAlphaOfLength(10)); timeSeriesIdBuilder.addLong("long", randomLong()); timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); - BytesRef tsidBytes = timeSeriesIdBuilder.build().toBytesRef(); - Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(tsidBytes); - BytesRef tsidParse = DocValueFormat.TIME_SERIES_ID.parseBytesRef(tsidFormat); - assertEquals(tsidBytes, tsidParse); + BytesRef expected = timeSeriesIdBuilder.buildTsidHash().toBytesRef(); + byte[] expectedBytes = new byte[expected.length]; + System.arraycopy(expected.bytes, 0, expectedBytes, 0, expected.length); + BytesRef actual = DocValueFormat.TIME_SERIES_ID.parseBytesRef(expected); + assertEquals(expected, actual); + Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(expected); + Object tsidBase64 = Base64.getUrlEncoder().withoutPadding().encodeToString(expectedBytes); + assertEquals(tsidFormat, tsidBase64); } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java new file mode 100644 index 0000000000000..850af7f85f76a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.TelemetryMetrics; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.hamcrest.MatcherAssert; +import org.junit.After; +import org.junit.Before; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; +import static org.elasticsearch.rest.action.search.SearchResponseMetrics.TOOK_DURATION_TOTAL_HISTOGRAM_NAME; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertScrollResponsesAndHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHitsWithoutFailures; +import static org.hamcrest.Matchers.greaterThan; + +public class SearchTookTimeTelemetryTests extends ESSingleNodeTestCase { + private static final String indexName = "test_search_metrics2"; + + @Override + protected boolean resetNodeAfterTest() { + return true; + } + + @Before + public void setUpIndex() throws Exception { + var num_primaries = randomIntBetween(1, 4); + createIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, num_primaries) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build() + ); + ensureGreen(indexName); + + prepareIndex(indexName).setId("1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("2").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); + } + + @After + private void afterTest() { + resetMeter(); + } + + @Override + protected Collection> getPlugins() { + return pluginList(TestTelemetryPlugin.class); + } + + public void testSimpleQuery() { + assertSearchHitsWithoutFailures(client().prepareSearch(indexName).setQuery(simpleQueryStringQuery("foo")), "1", "2"); + assertMetricsRecorded(); + } + + public void testScroll() { + assertScrollResponsesAndHitCount( + client(), + TimeValue.timeValueSeconds(60), + client().prepareSearch(indexName).setSize(1).setQuery(simpleQueryStringQuery("foo")), + 2, + (respNum, response) -> { + if (respNum <= 2) { + assertMetricsRecorded(); + } + resetMeter(); + } + ); + } + + private void assertMetricsRecorded() { + MatcherAssert.assertThat(getNumberOfLongHistogramMeasurements(TOOK_DURATION_TOTAL_HISTOGRAM_NAME), greaterThan(0)); + } + + private void resetMeter() { + getTestTelemetryPlugin().resetMeter(); + } + + private int getNumberOfLongHistogramMeasurements(String instrumentName) { + final List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(instrumentName); + return measurements.size(); + } + + private TestTelemetryPlugin getTestTelemetryPlugin() { + return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().get(0); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java index f638dd8e7c2b7..fe037415a9a3b 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java @@ -95,6 +95,7 @@ public void testSearchTransportMetricsScroll() throws InterruptedException { ); // getNumShards(indexName).numPrimaries assertScrollResponsesAndHitCount( + client(), TimeValue.timeValueSeconds(60), prepareSearch(indexName).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(1).setQuery(simpleQueryStringQuery("foo")), 2, diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java index 4acf66ff979ab..03913717992c9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java @@ -242,7 +242,7 @@ public void collect(int doc, long owningBucketOrd) throws IOException { assertTrue(timestamp.advanceExact(doc)); BytesRef latestTSID = tsid.lookupOrd(tsid.ordValue()); long latestTimestamp = timestamp.longValue(); - assertEquals(latestTSID, aggCtx.getTsid()); + assertEquals(latestTSID, aggCtx.getTsidHash()); assertEquals(latestTimestamp, aggCtx.getTimestamp()); if (currentTSID != null) { @@ -255,14 +255,14 @@ public void collect(int doc, long owningBucketOrd) throws IOException { currentTimestamp + "->" + latestTimestamp, timestampReverse ? latestTimestamp <= currentTimestamp : latestTimestamp >= currentTimestamp ); - assertEquals(currentTSIDord, aggCtx.getTsidOrd()); + assertEquals(currentTSIDord, aggCtx.getTsidHashOrd()); } else { - assertThat(aggCtx.getTsidOrd(), greaterThan(currentTSIDord)); + assertThat(aggCtx.getTsidHashOrd(), greaterThan(currentTSIDord)); } } currentTimestamp = latestTimestamp; currentTSID = BytesRef.deepCopyOf(latestTSID); - currentTSIDord = aggCtx.getTsidOrd(); + currentTSIDord = aggCtx.getTsidHashOrd(); total++; } }; diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index 2d19b269b9075..e52a638f566a1 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.List; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -78,7 +79,7 @@ protected KnnVectorQueryBuilder doCreateTestQueryBuilder() { for (int i = 0; i < vector.length; i++) { vector[i] = elementType().equals(DenseVectorFieldMapper.ElementType.BYTE) ? randomByte() : randomFloat(); } - int numCands = randomIntBetween(1, 1000); + int numCands = randomIntBetween(DEFAULT_SIZE, 1000); KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(fieldName, vector, numCands, randomBoolean() ? null : randomFloat()); if (randomBoolean()) { List filters = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java index 99bdfe58a2714..a077367604e5e 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java @@ -35,6 +35,7 @@ import java.util.Objects; import static java.util.Collections.emptyList; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -83,7 +84,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { @Override protected KnnSearchBuilder doParseInstance(XContentParser parser) throws IOException { - return KnnSearchBuilder.fromXContent(parser); + return KnnSearchBuilder.fromXContent(parser).build(DEFAULT_SIZE); } @Override @@ -101,20 +102,25 @@ protected KnnSearchBuilder mutateInstance(KnnSearchBuilder instance) { switch (random().nextInt(7)) { case 0: String newField = randomValueOtherThan(instance.field, () -> randomAlphaOfLength(5)); - return new KnnSearchBuilder(newField, instance.queryVector, instance.k, instance.numCands + 3, instance.similarity).boost( + return new KnnSearchBuilder(newField, instance.queryVector, instance.k, instance.numCands, instance.similarity).boost( instance.boost ); case 1: float[] newVector = randomValueOtherThan(instance.queryVector, () -> randomVector(5)); - return new KnnSearchBuilder(instance.field, newVector, instance.k + 3, instance.numCands, instance.similarity).boost( + return new KnnSearchBuilder(instance.field, newVector, instance.k, instance.numCands, instance.similarity).boost( instance.boost ); case 2: - return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k + 3, instance.numCands, instance.similarity) - .boost(instance.boost); + // given how the test instance is created, we have a 20-value gap between `k` and `numCands` so we SHOULD be safe + Integer newK = randomValueOtherThan(instance.k, () -> instance.k + ESTestCase.randomInt(10)); + return new KnnSearchBuilder(instance.field, instance.queryVector, newK, instance.numCands, instance.similarity).boost( + instance.boost + ); case 3: - return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k, instance.numCands + 3, instance.similarity) - .boost(instance.boost); + Integer newNumCands = randomValueOtherThan(instance.numCands, () -> instance.numCands + ESTestCase.randomInt(100)); + return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k, newNumCands, instance.similarity).boost( + instance.boost + ); case 4: return new KnnSearchBuilder(instance.field, instance.queryVector, instance.k, instance.numCands, instance.similarity) .addFilterQueries(instance.filterQueries) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 97275f7305b20..26f41b932f98f 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -168,6 +168,7 @@ import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.reservedstate.service.FileSettingsService; +import org.elasticsearch.rest.action.search.SearchResponseMetrics; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchService; @@ -2018,7 +2019,8 @@ protected void assertSnapshotOrGenericThread() { indexNameExpressionResolver, namedWriteableRegistry, EmptySystemIndices.INSTANCE.getExecutorSelector(), - new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()) + new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()), + new SearchResponseMetrics(TelemetryProvider.NOOP.getMeterRegistry()) ) ); actions.put( diff --git a/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/ApmIntegrationPlugin.java b/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/ApmIntegrationPlugin.java index 7ecdf253364f4..ea7a27018b4ca 100644 --- a/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/ApmIntegrationPlugin.java +++ b/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/ApmIntegrationPlugin.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -23,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class ApmIntegrationPlugin extends Plugin implements ActionPlugin { @@ -37,7 +39,8 @@ public List getRestHandlers( final IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter, final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster + final Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Collections.singletonList(testApmIntegrationRestHandler); } diff --git a/test/external-modules/die-with-dignity/src/main/java/org/elasticsearch/test/diewithdignity/DieWithDignityPlugin.java b/test/external-modules/die-with-dignity/src/main/java/org/elasticsearch/test/diewithdignity/DieWithDignityPlugin.java index c974551fbbc15..bca142bc9ac4b 100644 --- a/test/external-modules/die-with-dignity/src/main/java/org/elasticsearch/test/diewithdignity/DieWithDignityPlugin.java +++ b/test/external-modules/die-with-dignity/src/main/java/org/elasticsearch/test/diewithdignity/DieWithDignityPlugin.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -22,6 +23,7 @@ import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class DieWithDignityPlugin extends Plugin implements ActionPlugin { @@ -35,7 +37,8 @@ public List getRestHandlers( final IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter, final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster + final Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Collections.singletonList(new RestDieWithDignityAction()); } diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index 8d4b5ece98993..4e6e149b454e8 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.heap_attack; -import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; import org.apache.http.util.EntityUtils; @@ -21,7 +20,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -31,7 +29,6 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.After; import org.junit.Before; @@ -87,7 +84,7 @@ public void skipOnAborted() { public void testSortByManyLongsSuccess() throws IOException { initManyLongs(); Response response = sortByManyLongs(2000); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); + Map map = responseAsMap(response); ListMatcher columns = matchesList().item(matchesMap().entry("name", "a").entry("type", "long")) .item(matchesMap().entry("name", "b").entry("type", "long")); ListMatcher values = matchesList(); @@ -109,7 +106,7 @@ public void testSortByManyLongsTooMuchMemory() throws IOException { private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { ResponseException e = expectThrows(ResponseException.class, r); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(e.getResponse().getEntity()), false); + Map map = responseAsMap(e.getResponse()); logger.info("expected circuit breaker {}", map); assertMap( map, @@ -133,11 +130,8 @@ private Response sortByManyLongs(int count) throws IOException { */ public void testGroupOnSomeLongs() throws IOException { initManyLongs(); - Map map = XContentHelper.convertToMap( - JsonXContent.jsonXContent, - EntityUtils.toString(groupOnManyLongs(200).getEntity()), - false - ); + Response resp = groupOnManyLongs(200); + Map map = responseAsMap(resp); ListMatcher columns = matchesList().item(matchesMap().entry("name", "MAX(a)").entry("type", "long")); ListMatcher values = matchesList().item(List.of(9)); assertMap(map, matchesMap().entry("columns", columns).entry("values", values)); @@ -148,11 +142,8 @@ public void testGroupOnSomeLongs() throws IOException { */ public void testGroupOnManyLongs() throws IOException { initManyLongs(); - Map map = XContentHelper.convertToMap( - JsonXContent.jsonXContent, - EntityUtils.toString(groupOnManyLongs(5000).getEntity()), - false - ); + Response resp = groupOnManyLongs(5000); + Map map = responseAsMap(resp); ListMatcher columns = matchesList().item(matchesMap().entry("name", "MAX(a)").entry("type", "long")); ListMatcher values = matchesList().item(List.of(9)); assertMap(map, matchesMap().entry("columns", columns).entry("values", values)); @@ -180,7 +171,8 @@ private StringBuilder makeManyLongs(int count) { public void testSmallConcat() throws IOException { initSingleDocIndex(); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(concat(2).getEntity()), false); + Response resp = concat(2); + Map map = responseAsMap(resp); ListMatcher columns = matchesList().item(matchesMap().entry("name", "a").entry("type", "long")) .item(matchesMap().entry("name", "str").entry("type", "keyword")); ListMatcher values = matchesList().item(List.of(1, "1".repeat(100))); @@ -190,7 +182,7 @@ public void testSmallConcat() throws IOException { public void testHugeConcat() throws IOException { initSingleDocIndex(); ResponseException e = expectThrows(ResponseException.class, () -> concat(10)); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(e.getResponse().getEntity()), false); + Map map = responseAsMap(e.getResponse()); logger.info("expected request rejected {}", map); assertMap( map, @@ -216,7 +208,8 @@ private Response concat(int evals) throws IOException { */ public void testManyConcat() throws IOException { initManyLongs(); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(manyConcat(300).getEntity()), false); + Response resp = manyConcat(300); + Map map = responseAsMap(resp); ListMatcher columns = matchesList(); for (int s = 0; s < 300; s++) { columns = columns.item(matchesMap().entry("name", "str" + s).entry("type", "keyword")); @@ -267,7 +260,8 @@ private Response manyConcat(int strings) throws IOException { public void testManyEval() throws IOException { initManyLongs(); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(manyEval(1).getEntity()), false); + Response resp = manyEval(1); + Map map = responseAsMap(resp); ListMatcher columns = matchesList(); columns = columns.item(matchesMap().entry("name", "a").entry("type", "long")); columns = columns.item(matchesMap().entry("name", "b").entry("type", "long")); @@ -369,7 +363,7 @@ public void testFetchTooManyBigFields() throws IOException { */ private void fetchManyBigFields(int docs) throws IOException { Response response = query("{\"query\": \"FROM manybigfields | SORT f000 | LIMIT " + docs + "\"}", "columns"); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); + Map map = responseAsMap(response); ListMatcher columns = matchesList(); for (int f = 0; f < 1000; f++) { columns = columns.item(matchesMap().entry("name", "f" + String.format(Locale.ROOT, "%03d", f)).entry("type", "keyword")); @@ -381,7 +375,7 @@ public void testAggMvLongs() throws IOException { int fieldValues = 100; initMvLongsIndex(1, 3, fieldValues); Response response = aggMvLongs(3); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); + Map map = responseAsMap(response); ListMatcher columns = matchesList().item(matchesMap().entry("name", "MAX(f00)").entry("type", "long")) .item(matchesMap().entry("name", "f00").entry("type", "long")) .item(matchesMap().entry("name", "f01").entry("type", "long")) @@ -406,7 +400,7 @@ public void testFetchMvLongs() throws IOException { int fields = 100; initMvLongsIndex(100, fields, 1000); Response response = fetchMvLongs(); - Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); + Map map = responseAsMap(response); ListMatcher columns = matchesList(); for (int f = 0; f < fields; f++) { columns = columns.item(matchesMap().entry("name", String.format(Locale.ROOT, "f%02d", f)).entry("type", "long")); @@ -570,8 +564,8 @@ public void assertRequestBreakerEmpty() throws Exception { return; } assertBusy(() -> { - HttpEntity entity = adminClient().performRequest(new Request("GET", "/_nodes/stats")).getEntity(); - Map stats = XContentHelper.convertToMap(XContentType.JSON.xContent(), entity.getContent(), false); + Response response = adminClient().performRequest(new Request("GET", "/_nodes/stats")); + Map stats = responseAsMap(response); Map nodes = (Map) stats.get("nodes"); for (Object n : nodes.values()) { Map node = (Map) n; diff --git a/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java b/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java index 77e0c3b3e0821..224a3eccfef46 100644 --- a/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java +++ b/test/external-modules/esql-heap-attack/src/main/java/org/elasticsearch/test/esql/heap_attack/HeapAttackPlugin.java @@ -22,12 +22,14 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class HeapAttackPlugin extends Plugin implements ActionPlugin { @@ -40,7 +42,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestTriggerOutOfMemoryAction()); } diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java index bb88aad387a0d..54ef53b8969e1 100644 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java +++ b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/SeekTrackerPlugin.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexModule; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -28,6 +29,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class SeekTrackerPlugin extends Plugin implements ActionPlugin { @@ -77,7 +79,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (enabled) { return Collections.singletonList(new RestSeekStatsAction()); diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java index 336b888dd7d3c..2698d96ab7ab9 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java @@ -12,6 +12,7 @@ import com.sun.net.httpserver.HttpHandler; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.CompositeBytesReference; @@ -31,6 +32,7 @@ import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -74,14 +76,21 @@ public S3HttpHandler(final String bucket, @Nullable final String basePath) { @Override public void handle(final HttpExchange exchange) throws IOException { - final String request = exchange.getRequestMethod() + " " + exchange.getRequestURI().toString(); + // Remove custom query parameters before processing the request. This simulates how S3 ignores them. + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html#LogFormatCustom + final RequestComponents requestComponents = parseRequestComponents( + exchange.getRequestMethod() + " " + exchange.getRequestURI().toString() + ); + final String request = requestComponents.request(); + onCustomQueryParameters(requestComponents.customQueryParameters); + if (request.startsWith("GET") || request.startsWith("HEAD") || request.startsWith("DELETE")) { int read = exchange.getRequestBody().read(); assert read == -1 : "Request body should have been empty but saw [" + read + "]"; } try { if (Regex.simpleMatch("HEAD /" + path + "/*", request)) { - final BytesReference blob = blobs.get(exchange.getRequestURI().getPath()); + final BytesReference blob = blobs.get(requestComponents.path); if (blob == null) { exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); } else { @@ -89,8 +98,7 @@ public void handle(final HttpExchange exchange) throws IOException { } } else if (Regex.simpleMatch("GET /" + bucket + "/?uploads&prefix=*", request)) { final Map params = new HashMap<>(); - RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params); - + RestUtils.decodeQueryString(request, request.indexOf('?') + 1, params); final var prefix = params.get("prefix"); final var uploadsList = new StringBuilder(); @@ -120,10 +128,7 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.getResponseBody().write(response); } else if (Regex.simpleMatch("POST /" + path + "/*?uploads", request)) { - final var upload = new MultipartUpload( - UUIDs.randomBase64UUID(), - exchange.getRequestURI().getPath().substring(bucket.length() + 2) - ); + final var upload = new MultipartUpload(UUIDs.randomBase64UUID(), requestComponents.path.substring(bucket.length() + 2)); uploads.put(upload.getUploadId(), upload); final var uploadResult = new StringBuilder(); @@ -141,7 +146,7 @@ public void handle(final HttpExchange exchange) throws IOException { } else if (Regex.simpleMatch("PUT /" + path + "/*?uploadId=*&partNumber=*", request)) { final Map params = new HashMap<>(); - RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params); + RestUtils.decodeQueryString(request, request.indexOf('?') + 1, params); final var upload = uploads.get(params.get("uploadId")); if (upload == null) { @@ -154,15 +159,14 @@ public void handle(final HttpExchange exchange) throws IOException { } } else if (Regex.simpleMatch("POST /" + path + "/*?uploadId=*", request)) { - final Map params = new HashMap<>(); - RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params); + RestUtils.decodeQueryString(request, request.indexOf('?') + 1, params); final var upload = uploads.remove(params.get("uploadId")); if (upload == null) { exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); } else { final var blobContents = upload.complete(extractPartEtags(Streams.readFully(exchange.getRequestBody()))); - blobs.put(exchange.getRequestURI().getPath(), blobContents); + blobs.put(requestComponents.path, blobContents); byte[] response = ("\n" + "\n" @@ -170,7 +174,7 @@ public void handle(final HttpExchange exchange) throws IOException { + bucket + "\n" + "" - + exchange.getRequestURI().getPath() + + requestComponents.path + "\n" + "").getBytes(StandardCharsets.UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/xml"); @@ -179,19 +183,19 @@ public void handle(final HttpExchange exchange) throws IOException { } } else if (Regex.simpleMatch("DELETE /" + path + "/*?uploadId=*", request)) { final Map params = new HashMap<>(); - RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params); + RestUtils.decodeQueryString(request, request.indexOf('?') + 1, params); final var upload = uploads.remove(params.get("uploadId")); exchange.sendResponseHeaders((upload == null ? RestStatus.NOT_FOUND : RestStatus.NO_CONTENT).getStatus(), -1); } else if (Regex.simpleMatch("PUT /" + path + "/*", request)) { final Tuple blob = parseRequestBody(exchange); - blobs.put(exchange.getRequestURI().toString(), blob.v2()); + blobs.put(requestComponents.uri(), blob.v2()); exchange.getResponseHeaders().add("ETag", blob.v1()); exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1); } else if (Regex.simpleMatch("GET /" + bucket + "/?prefix=*", request)) { final Map params = new HashMap<>(); - RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params); + RestUtils.decodeQueryString(request, request.indexOf('?') + 1, params); if (params.get("list-type") != null) { throw new AssertionError("Test must be adapted for GET Bucket (List Objects) Version 2"); } @@ -240,7 +244,7 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.getResponseBody().write(response); } else if (Regex.simpleMatch("GET /" + path + "/*", request)) { - final BytesReference blob = blobs.get(exchange.getRequestURI().toString()); + final BytesReference blob = blobs.get(requestComponents.uri()); if (blob != null) { final String range = exchange.getRequestHeaders().getFirst("Range"); if (range == null) { @@ -271,7 +275,7 @@ public void handle(final HttpExchange exchange) throws IOException { int deletions = 0; for (Iterator> iterator = blobs.entrySet().iterator(); iterator.hasNext();) { Map.Entry blob = iterator.next(); - if (blob.getKey().startsWith(exchange.getRequestURI().toString())) { + if (blob.getKey().startsWith(requestComponents.uri())) { iterator.remove(); deletions++; } @@ -311,6 +315,42 @@ public Map blobs() { return blobs; } + protected void onCustomQueryParameters(final Map> params) {} + + public static RequestComponents parseRequestComponents(final String request) { + final int spacePos = request.indexOf(' '); + final String method = request.substring(0, spacePos); + final String uriString = request.substring(spacePos + 1); + final int questsionMarkPos = uriString.indexOf('?'); + // AWS s3 allows the same custom query parameter to be specified multiple times + final Map> customQueryParameters = new HashMap<>(); + if (questsionMarkPos == -1) { + return new RequestComponents(method, uriString, "", customQueryParameters); + } else { + final String queryString = uriString.substring(questsionMarkPos + 1); + final ArrayList queryParameters = new ArrayList<>(); + Arrays.stream(Strings.tokenizeToStringArray(queryString, "&")).forEach(param -> { + if (param.startsWith("x-")) { + final int equalPos = param.indexOf("="); + customQueryParameters.computeIfAbsent(param.substring(0, equalPos), k -> new ArrayList<>()) + .add(param.substring(equalPos + 1)); + } else { + queryParameters.add(param); + } + }); + return new RequestComponents( + method, + uriString.substring(0, questsionMarkPos), + Strings.collectionToDelimitedString(queryParameters, "&"), + customQueryParameters + ); + } + } + + public static String getRawRequestString(final HttpExchange exchange) { + return exchange.getRequestMethod() + " " + exchange.getRequestURI(); + } + private static final Pattern chunkSignaturePattern = Pattern.compile("^([0-9a-z]+);chunk-signature=([^\\r\\n]*)$"); private static Tuple parseRequestBody(final HttpExchange exchange) throws IOException { @@ -475,4 +515,19 @@ public static void sendError(final HttpExchange exchange, final RestStatus statu MultipartUpload getUpload(String uploadId) { return uploads.get(uploadId); } + + public record RequestComponents(String method, String path, String query, Map> customQueryParameters) { + + public String request() { + return method + " " + uri(); + } + + public String uri() { + if (query.isEmpty()) { + return path; + } else { + return path + "?" + query; + } + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index d0b30bff92f3e..3a47e0885f2d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -124,7 +124,20 @@ public static DataStream newInstance( @Nullable DataStreamLifecycle lifecycle, List failureStores ) { - return new DataStream(name, indices, generation, metadata, false, replicated, false, false, null, lifecycle, false, failureStores); + return new DataStream( + name, + indices, + generation, + metadata, + false, + replicated, + false, + false, + null, + lifecycle, + failureStores.size() > 0, + failureStores + ); } public static String getLegacyDefaultBackingIndexName( @@ -169,6 +182,25 @@ public static IndexMetadata.Builder createBackingIndex(String dataStreamName, in .numberOfReplicas(NUMBER_OF_REPLICAS); } + public static IndexMetadata.Builder createFirstFailureStore(String dataStreamName) { + return createFailureStore(dataStreamName, 1, System.currentTimeMillis()); + } + + public static IndexMetadata.Builder createFirstFailureStore(String dataStreamName, long epochMillis) { + return createFailureStore(dataStreamName, 1, epochMillis); + } + + public static IndexMetadata.Builder createFailureStore(String dataStreamName, int generation) { + return createFailureStore(dataStreamName, generation, System.currentTimeMillis()); + } + + public static IndexMetadata.Builder createFailureStore(String dataStreamName, int generation, long epochMillis) { + return IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, generation, epochMillis)) + .settings(SETTINGS) + .numberOfShards(NUMBER_OF_SHARDS) + .numberOfReplicas(NUMBER_OF_REPLICAS); + } + public static IndexMetadata.Builder getIndexMetadataBuilderForIndex(Index index) { return IndexMetadata.builder(index.getName()) .settings(Settings.builder().put(SETTINGS.build()).put(SETTING_INDEX_UUID, index.getUUID())) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java similarity index 98% rename from server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java rename to test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 7b91c84a05c53..81848b5a50114 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.script.DoubleFieldScript; import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.Script; @@ -45,7 +44,7 @@ public abstract class NumberFieldMapperTests extends MapperTestCase { /** * @return a List of OutOfRangeSpec to test for this number type */ - protected abstract List outOfRangeSpecs(); + protected abstract List outOfRangeSpecs(); /** * @return an appropriate value to use for a missing value for this number type @@ -234,7 +233,7 @@ public void testNullValue() throws IOException { } public void testOutOfRangeValues() throws IOException { - for (OutOfRangeSpec item : outOfRangeSpecs()) { + for (NumberTypeOutOfRangeSpec item : outOfRangeSpecs()) { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", item.type.typeName()))); Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(item::write))); assertThat( @@ -317,7 +316,7 @@ public void testMetricAndDocvalues() { } @Override - protected final Object generateRandomInputValue(MappedFieldType ft) { + protected Object generateRandomInputValue(MappedFieldType ft) { Number n = randomNumber(); return randomBoolean() ? n : n.toString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberTypeOutOfRangeSpec.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberTypeOutOfRangeSpec.java new file mode 100644 index 0000000000000..cf8c9a06088db --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberTypeOutOfRangeSpec.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.math.BigInteger; + +public class NumberTypeOutOfRangeSpec { + + final NumberFieldMapper.NumberType type; + final Object value; + final String message; + + public static NumberTypeOutOfRangeSpec of(NumberFieldMapper.NumberType t, Object v, String m) { + return new NumberTypeOutOfRangeSpec(t, v, m); + } + + NumberTypeOutOfRangeSpec(NumberFieldMapper.NumberType t, Object v, String m) { + type = t; + value = v; + message = m; + } + + public void write(XContentBuilder b) throws IOException { + if (value instanceof BigInteger) { + b.rawField("field", new ByteArrayInputStream(value.toString().getBytes("UTF-8")), XContentType.JSON); + } else { + b.field("field", value); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 4e953d02e4d81..d4c238322e28a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -34,8 +34,12 @@ public TestDocumentParserContext() { this(MappingLookup.EMPTY, null); } + public TestDocumentParserContext(Settings settings) { + this(MappingLookup.EMPTY, null, null, settings); + } + public TestDocumentParserContext(XContentParser parser) { - this(MappingLookup.EMPTY, null, parser); + this(MappingLookup.EMPTY, null, parser, Settings.EMPTY); } /** @@ -43,10 +47,10 @@ public TestDocumentParserContext(XContentParser parser) { * that depend on them are called while executing tests. */ public TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse source) { - this(mappingLookup, source, null); + this(mappingLookup, source, null, Settings.EMPTY); } - private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse source, XContentParser parser) { + private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse source, XContentParser parser, Settings settings) { super( mappingLookup, new MappingParserContext( @@ -58,7 +62,7 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou () -> null, null, (type, name) -> Lucene.STANDARD_ANALYZER, - MapperTestCase.createIndexSettings(IndexVersion.current(), Settings.EMPTY), + MapperTestCase.createIndexSettings(IndexVersion.current(), settings), null ), source, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java similarity index 100% rename from server/src/test/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java rename to test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index be26eabe308ba..273859486eeda 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -101,6 +101,7 @@ import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.index.mapper.PassThroughObjectMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -200,6 +201,7 @@ public abstract class AggregatorTestCase extends ESTestCase { SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported NestedObjectMapper.CONTENT_TYPE, // TODO support for nested + PassThroughObjectMapper.CONTENT_TYPE, // TODO support for passthrough CompletionFieldMapper.CONTENT_TYPE, // TODO support completion FieldAliasMapper.CONTENT_TYPE // TODO support alias ); @@ -383,7 +385,12 @@ public void onCache(ShardId shardId, Accountable accountable) {} () -> true, valuesSourceRegistry, emptyMap() - ); + ) { + @Override + public Iterable dimensionFields() { + return Arrays.stream(fieldTypes).filter(MappedFieldType::isDimension).toList(); + } + }; AggregationContext context = new ProductionAggregationContext( Optional.ofNullable(analysisModule).map(AnalysisModule::getAnalysisRegistry).orElse(null), diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryVectorBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryVectorBuilderTestCase.java index ab5e3a7555214..b6e5c7161edc8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryVectorBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryVectorBuilderTestCase.java @@ -25,11 +25,13 @@ import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParser; import org.junit.Before; import java.io.IOException; import java.util.List; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -68,12 +70,21 @@ protected T createTestInstance(float[] expected) { return createTestInstance(); } + protected KnnSearchBuilder parseKnnSearchBuilder(XContentParser parser) throws IOException { + return KnnSearchBuilder.fromXContent(parser).build(DEFAULT_SIZE); + } + public final void testKnnSearchBuilderXContent() throws Exception { AbstractXContentTestCase.XContentTester tester = AbstractXContentTestCase.xContentTester( this::createParser, - () -> new KnnSearchBuilder(randomAlphaOfLength(10), createTestInstance(), 5, 10, randomBoolean() ? null : randomFloat()), + () -> new KnnSearchBuilder.Builder().field(randomAlphaOfLength(10)) + .queryVectorBuilder(createTestInstance()) + .k(5) + .numCandidates(10) + .similarity(randomBoolean() ? null : randomFloat()) + .build(DEFAULT_SIZE), getToXContentParams(), - KnnSearchBuilder::fromXContent + this::parseKnnSearchBuilder ); tester.test(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 65b28ad874431..82e02ec1db70f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1649,7 +1649,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma Set indices = new HashSet<>(); builders = new ArrayList<>(builders); for (IndexRequestBuilder builder : builders) { - indices.add(builder.request().index()); + indices.add(builder.getIndex()); } Set> bogusIds = new HashSet<>(); // (index, type, id) if (random.nextBoolean() && builders.isEmpty() == false && dummyDocuments) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index 99cf880a83604..04c06b8b5d1cf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -22,14 +22,15 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse; import org.elasticsearch.action.support.master.IsAcknowledgedSupplier; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -50,6 +51,7 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.hamcrest.Matcher; +import org.hamcrest.Matchers; import java.io.IOException; import java.nio.file.Files; @@ -69,7 +71,6 @@ import static org.apache.lucene.tests.util.LuceneTestCase.expectThrows; import static org.apache.lucene.tests.util.LuceneTestCase.expectThrowsAnyOf; -import static org.elasticsearch.test.ESIntegTestCase.clearScroll; import static org.elasticsearch.test.ESIntegTestCase.client; import static org.elasticsearch.test.LambdaMatchers.transformedArrayItemsMatch; import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; @@ -387,6 +388,7 @@ public static void assertRes * respNum starts at 1, which contains the resp from the initial request. */ public static void assertScrollResponsesAndHitCount( + Client client, TimeValue keepAlive, SearchRequestBuilder searchRequestBuilder, int expectedTotalHitCount, @@ -402,23 +404,20 @@ public static void assertScrollResponsesAndHitCount( retrievedDocsCount += scrollResponse.getHits().getHits().length; responseConsumer.accept(responses.size(), scrollResponse); while (scrollResponse.getHits().getHits().length > 0) { - scrollResponse = prepareScrollSearch(scrollResponse.getScrollId(), keepAlive).get(); + scrollResponse = client.prepareSearchScroll(scrollResponse.getScrollId()).setScroll(keepAlive).get(); responses.add(scrollResponse); assertThat(scrollResponse.getHits().getTotalHits().value, equalTo((long) expectedTotalHitCount)); retrievedDocsCount += scrollResponse.getHits().getHits().length; responseConsumer.accept(responses.size(), scrollResponse); } } finally { - clearScroll(scrollResponse.getScrollId()); + ClearScrollResponse clearResponse = client.prepareClearScroll().setScrollIds(Arrays.asList(scrollResponse.getScrollId())).get(); responses.forEach(SearchResponse::decRef); + assertThat(clearResponse.isSucceeded(), Matchers.equalTo(true)); } assertThat(retrievedDocsCount, equalTo(expectedTotalHitCount)); } - public static SearchScrollRequestBuilder prepareScrollSearch(String scrollId, TimeValue timeout) { - return client().prepareSearchScroll(scrollId).setScroll(timeout); - } - public static void assertResponse(ActionFuture responseFuture, Consumer consumer) throws ExecutionException, InterruptedException { var res = responseFuture.get(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index a3427b3778b0a..1860283515c9d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1899,20 +1899,36 @@ protected static Map getAlias(final String index, final String a } protected static Map getAsMap(final String endpoint) throws IOException { - return getAsMap(client(), endpoint); + return getAsMap(client(), endpoint, false); + } + + protected static Map getAsOrderedMap(final String endpoint) throws IOException { + return getAsMap(client(), endpoint, true); } protected static Map getAsMap(RestClient client, final String endpoint) throws IOException { + return getAsMap(client, endpoint, false); + } + + private static Map getAsMap(RestClient client, final String endpoint, final boolean ordered) throws IOException { Response response = client.performRequest(new Request("GET", endpoint)); - return responseAsMap(response); + return responseAsMap(response, ordered); } protected static Map responseAsMap(Response response) throws IOException { + return responseAsMap(response, false); + } + + protected static Map responseAsOrderedMap(Response response) throws IOException { + return responseAsMap(response, true); + } + + private static Map responseAsMap(Response response, boolean ordered) throws IOException { XContentType entityContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue()); Map responseEntity = XContentHelper.convertToMap( entityContentType.xContent(), response.getEntity().getContent(), - false + ordered ); assertNotNull(responseEntity); return responseEntity; diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java index 7633d38d7ebba..2a1d9f8c44c53 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregator.java @@ -95,11 +95,14 @@ protected MultiTermsAggregator( partiallyBuiltBucketComparator = order == null ? null : order.partiallyBuiltBucketComparator(b -> b.bucketOrd, this); this.formats = formats; this.showTermDocCountError = showTermDocCountError; - if (subAggsNeedScore() && descendsFromNestedAggregator(parent)) { + if (subAggsNeedScore() && descendsFromNestedAggregator(parent) || context.isInSortOrderExecutionRequired()) { /** * Force the execution to depth_first because we need to access the score of * nested documents in a sub-aggregation and we are not able to generate this score * while replaying deferred documents. + * + * We also force depth_first for time-series aggs executions since they need to be visited in a particular order (index + * sort order) which might be changed by the breadth_first execution. */ this.collectMode = SubAggCollectionMode.DEPTH_FIRST; } else { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java index 5eefa7cfc56fa..f153a397f9f58 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java @@ -120,10 +120,10 @@ public void collect(int doc, long bucket) throws IOException { startTimes = bigArrays().grow(startTimes, bucket + 1); endTimes = bigArrays().grow(endTimes, bucket + 1); resetCompensations = bigArrays().grow(resetCompensations, bucket + 1); - if (currentTsid != aggCtx.getTsidOrd()) { + if (currentTsid != aggCtx.getTsidHashOrd()) { // if we're on a new tsid then we need to calculate the last bucket calculateLastBucket(); - currentTsid = aggCtx.getTsidOrd(); + currentTsid = aggCtx.getTsidHashOrd(); } else { // if we're in a new bucket but in the same tsid then we update the // timestamp and last value before we calculate the last bucket diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index b226f0f23b7ff..885e02a8b5e6a 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -11,14 +11,18 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -63,8 +67,7 @@ public void testSimple() throws IOException { closeTo(206.0 / 4000.0 * MILLIS_IN_SECOND, 0.00001) ); }; - AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field")) - .withSplitLeavesIntoSeperateAggregators(false); + AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"), dimensionField("dim")); testCase(iw -> { iw.addDocuments(docs(1000, "1", 15, 37, 60, /*reset*/ 14)); iw.addDocuments(docs(1000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40)); @@ -103,7 +106,7 @@ public void testNestedWithinDateHistogram() throws IOException { } }; - AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field")) + AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"), dimensionField("dim")) .withSplitLeavesIntoSeperateAggregators(false); testCase(iw -> { iw.addDocuments(docs(2000, "1", 15, 37, 60, /*reset*/ 14)); @@ -154,7 +157,7 @@ private List docs(long startTimestamp, String dim, long... values) thr List documents = new ArrayList<>(); for (int i = 0; i < values.length; i++) { - documents.add(doc(startTimestamp + (i * 1000L), tsid(dim), values[i])); + documents.add(doc(startTimestamp + (i * 1000L), tsid(dim), values[i], dim)); } return documents; } @@ -162,17 +165,25 @@ private List docs(long startTimestamp, String dim, long... values) thr private static BytesReference tsid(String dim) throws IOException { TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); idBuilder.addString("dim", dim); - return idBuilder.build(); + return idBuilder.buildTsidHash(); } - private Document doc(long timestamp, BytesReference tsid, long counterValue) { + private Document doc(long timestamp, BytesReference tsid, long counterValue, String dim) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("@timestamp", timestamp)); doc.add(new SortedDocValuesField("_tsid", tsid.toBytesRef())); doc.add(new NumericDocValuesField("counter_field", counterValue)); + doc.add(new SortedDocValuesField("dim", new BytesRef(dim))); return doc; } + private MappedFieldType dimensionField(String name) { + return new KeywordFieldMapper.Builder(name, IndexVersion.current()).dimension(true) + .docValues(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(); + } + private MappedFieldType counterField(String name) { return new NumberFieldMapper.NumberFieldType( name, diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index 195d00169840a..0f80df5e6db50 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -27,6 +28,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.xpack.core.async.AsyncTaskMaintenanceService.ASYNC_SEARCH_CLEANUP_INTERVAL_SETTING; @@ -51,7 +53,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList( new RestSubmitAsyncSearchAction(restController.getSearchUsageHolder(), namedWriteableRegistry), diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java index 3b84ce4ea881c..7e9e83e616f61 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.NodeRoles; import org.elasticsearch.xpack.autoscaling.AbstractFrozenAutoscalingIntegTestCase; @@ -83,7 +82,6 @@ protected Collection> nodePlugins() { ); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102405") public void testZeroToOne() throws Exception { internalCluster().startMasterOnlyNode(); setupRepoAndPolicy(); @@ -91,7 +89,7 @@ public void testZeroToOne() throws Exception { internalCluster().startNode(NodeRoles.onlyRole(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE)); internalCluster().startNode(NodeRoles.onlyRole(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE)); // create an ignored snapshot to initialize the latest-N file. - final SnapshotInfo snapshotInfo = createFullSnapshot(fsRepoName, snapshotName); + createFullSnapshot(fsRepoName, snapshotName); Phase hotPhase = new Phase("hot", TimeValue.ZERO, Collections.emptyMap()); Phase frozenPhase = new Phase( @@ -111,9 +109,9 @@ public void testZeroToOne() throws Exception { .build(); CreateIndexResponse res = indicesAdmin().prepareCreate(INDEX_NAME).setSettings(settings).get(); assertTrue(res.isAcknowledged()); - logger.info("created index"); + logger.info("-> created index"); - assertBusy(() -> { assertMinimumCapacity(capacity().results().get("frozen").requiredCapacity().total()); }); + assertBusy(() -> assertMinimumCapacity(capacity().results().get("frozen").requiredCapacity().total())); assertMinimumCapacity(capacity().results().get("frozen").requiredCapacity().node()); assertThat( @@ -134,14 +132,16 @@ public void testZeroToOne() throws Exception { // verify that SearchableSnapshotAction uses WaitForDataTierStep and that it waits. assertThat(indices(), not(arrayContaining(PARTIAL_INDEX_NAME))); - logger.info("starting dedicated frozen node"); + logger.info("-> starting dedicated frozen node"); internalCluster().startNode(NodeRoles.onlyRole(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)); + // we've seen a case where bootstrapping a node took just over 60 seconds in the test environment, so using an (excessive) 90 + // seconds max wait time to avoid flakiness assertBusy(() -> { String[] indices = indices(); assertThat(indices, arrayContaining(PARTIAL_INDEX_NAME)); assertThat(indices, not(arrayContaining(INDEX_NAME))); - }, 60, TimeUnit.SECONDS); + }, 90, TimeUnit.SECONDS); ensureGreen(); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java index 60220391a2165..88bd978b6f416 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.license.License; import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.plugins.ActionPlugin; @@ -61,6 +62,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -136,7 +138,8 @@ public List getRestHandlers( final IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter, final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster + final Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of( new RestGetAutoscalingCapacityHandler(), diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java index 66a6cf4dbd949..049197edd97df 100644 --- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java +++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java @@ -423,10 +423,15 @@ public void testMassiveDecay() throws IOException { * @throws IOException */ public void testGetMultiThreaded() throws IOException { - int threads = between(2, 10); - int regionCount = between(1, 20); + final int threads = between(2, 10); + final int regionCount = between(1, 20); + final boolean incRef = randomBoolean(); // if we have enough regions, a get should always have a result (except for explicit evict interference) - final boolean allowAlreadyClosed = regionCount < threads; + // if we incRef, we risk the eviction racing against that, leading to no available region, so allow + // the already closed exception in that case. + final boolean allowAlreadyClosed = regionCount < threads || incRef; + + logger.info("{} {} {}", threads, regionCount, allowAlreadyClosed); Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(regionCount * 100L)).getStringRep()) @@ -466,7 +471,7 @@ public void testGetMultiThreaded() throws IOException { assert allowAlreadyClosed || e.getMessage().equals("evicted during free region allocation") : e; throw e; } - if (cacheFileRegion.tryIncRef()) { + if (incRef && cacheFileRegion.tryIncRef()) { if (yield[i] == 0) { Thread.yield(); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index 4a3a92aa80bc8..efea78ee4730a 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; @@ -118,6 +119,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -263,7 +265,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (enabled == false) { return emptyList(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 086ab7d842eb1..1313e5781f122 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -319,6 +319,7 @@ public void testDynamicIndexSettingsAreClassified() { // These fields need to be replicated otherwise documents that can be indexed in the leader index cannot // be indexed in the follower index: replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 31c772f96f889..72b8a8b9e4d98 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Booleans; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; @@ -125,6 +126,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.LongSupplier; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -388,7 +390,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { List handlers = new ArrayList<>(); handlers.add(new RestXPackInfoAction()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResponse.java index b31544a1921a6..f6a9bd8474838 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncResponse.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.RefCounted; -public interface AsyncResponse> extends Writeable, RefCounted { +public interface AsyncResponse> extends Writeable, RefCounted { /** * When this response will expire as a timestamp in milliseconds since epoch. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java new file mode 100644 index 0000000000000..1ede0386ce314 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class ChunkedSparseEmbeddingResults implements ChunkedInferenceServiceResults { + + public static final String NAME = "chunked_sparse_embedding_results"; + + public static ChunkedSparseEmbeddingResults ofMlResult(ChunkedTextExpansionResults mlInferenceResults) { + return new ChunkedSparseEmbeddingResults(mlInferenceResults.getChunks()); + } + + private final List chunkedResults; + + public ChunkedSparseEmbeddingResults(List chunks) { + this.chunkedResults = chunks; + } + + public ChunkedSparseEmbeddingResults(StreamInput in) throws IOException { + this.chunkedResults = in.readCollectionAsList(ChunkedTextExpansionResults.ChunkedResult::new); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("sparse_embedding_chunk"); + for (ChunkedTextExpansionResults.ChunkedResult chunk : chunkedResults) { + chunk.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(chunkedResults); + } + + @Override + public List transformToCoordinationFormat() { + throw new UnsupportedOperationException("Chunked results are not returned in the coordindated action"); + } + + @Override + public List transformToLegacyFormat() { + throw new UnsupportedOperationException("Chunked results are not returned in the legacy format"); + } + + @Override + public Map asMap() { + throw new UnsupportedOperationException("Chunked results are not returned in the a map format"); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ChunkedSparseEmbeddingResults that = (ChunkedSparseEmbeddingResults) o; + return Objects.equals(chunkedResults, that.chunkedResults); + } + + @Override + public int hashCode() { + return Objects.hash(chunkedResults); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java new file mode 100644 index 0000000000000..3b3b0e7539bf7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class ChunkedTextEmbeddingResults implements ChunkedInferenceServiceResults { + + public static final String NAME = "chunked_text_embedding_service_results"; + + public static ChunkedTextEmbeddingResults ofMlResult( + org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults mlInferenceResults + ) { + return new ChunkedTextEmbeddingResults(mlInferenceResults.getChunks()); + } + + private final List chunks; + + public ChunkedTextEmbeddingResults( + List chunks + ) { + this.chunks = chunks; + } + + public ChunkedTextEmbeddingResults(StreamInput in) throws IOException { + this.chunks = in.readCollectionAsList( + org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults.EmbeddingChunk::new + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("text_embedding_chunk"); + for (var embedding : chunks) { + embedding.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(chunks); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public List transformToCoordinationFormat() { + throw new UnsupportedOperationException("Chunked results are not returned in the coordindated action"); + } + + @Override + public List transformToLegacyFormat() { + throw new UnsupportedOperationException("Chunked results are not returned in the legacy format"); + } + + @Override + public Map asMap() { + throw new UnsupportedOperationException("Chunked results are not returned in the a map format"); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ChunkedTextEmbeddingResults that = (ChunkedTextEmbeddingResults) o; + return Objects.equals(chunks, that.chunks); + } + + @Override + public int hashCode() { + return Objects.hash(chunks); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java index ffaa8489929ff..1b365bd96d834 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersion.java @@ -289,7 +289,7 @@ public static Tuple getMinMaxMlConfigVersion(D if (mlConfigVersion.after(maxMlConfigVersion)) { maxMlConfigVersion = mlConfigVersion; } - } catch (IllegalArgumentException e) { + } catch (IllegalStateException e) { // This means we encountered a node that is after 8.10.0 but has the ML plugin disabled - ignore it } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java index bd03913290e0c..2d7f1c4b52301 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java @@ -104,6 +104,7 @@ public static Request.Builder parseRequest(String id, XContentParser parser) { // input and so cannot construct a document. private final List textInput; private TrainedModelPrefixStrings.PrefixType prefixType = TrainedModelPrefixStrings.PrefixType.NONE; + private boolean chunkResults = false; public static Request forDocs(String id, InferenceConfigUpdate update, List> docs, TimeValue inferenceTimeout) { return new Request( @@ -163,6 +164,11 @@ public Request(StreamInput in) throws IOException { } else { prefixType = TrainedModelPrefixStrings.PrefixType.NONE; } + if (in.getTransportVersion().onOrAfter(TransportVersions.NLP_DOCUMENT_CHUNKING_ADDED)) { + chunkResults = in.readBoolean(); + } else { + chunkResults = false; + } } public String getId() { @@ -215,6 +221,14 @@ public TrainedModelPrefixStrings.PrefixType getPrefixType() { return prefixType; } + public boolean isChunkResults() { + return chunkResults; + } + + public void setChunkResults(boolean chunkResults) { + this.chunkResults = chunkResults; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = super.validate(); @@ -244,6 +258,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.ML_TRAINED_MODEL_PREFIX_STRINGS_ADDED)) { out.writeEnum(prefixType); } + if (out.getTransportVersion().onOrAfter(TransportVersions.NLP_DOCUMENT_CHUNKING_ADDED)) { + out.writeBoolean(chunkResults); + } } @Override @@ -262,12 +279,13 @@ public boolean equals(Object o) { && Objects.equals(inferenceTimeout, that.inferenceTimeout) && Objects.equals(highPriority, that.highPriority) && Objects.equals(textInput, that.textInput) - && (prefixType == that.prefixType); + && (prefixType == that.prefixType) + && (chunkResults == that.chunkResults); } @Override public int hashCode() { - return Objects.hash(id, update, docs, inferenceTimeout, highPriority, textInput, prefixType); + return Objects.hash(id, update, docs, inferenceTimeout, highPriority, textInput, prefixType, chunkResults); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java index 00587936848f8..9bcc443f6d7b0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java @@ -20,6 +20,8 @@ import org.elasticsearch.xpack.core.ml.inference.preprocessing.PreProcessor; import org.elasticsearch.xpack.core.ml.inference.preprocessing.StrictlyParsedPreProcessor; import org.elasticsearch.xpack.core.ml.inference.preprocessing.TargetMeanEncoding; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.FillMaskResults; @@ -671,6 +673,13 @@ public List getNamedWriteables() { TextSimilarityInferenceResults::new ) ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(InferenceResults.class, ChunkedTextEmbeddingResults.NAME, ChunkedTextEmbeddingResults::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(InferenceResults.class, ChunkedTextExpansionResults.NAME, ChunkedTextExpansionResults::new) + ); + // Inference Configs namedWriteables.add( new NamedWriteableRegistry.Entry(InferenceConfig.class, ClassificationConfig.NAME.getPreferredName(), ClassificationConfig::new) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedNlpInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedNlpInferenceResults.java new file mode 100644 index 0000000000000..d0505d66f6e16 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedNlpInferenceResults.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.results; + +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +public abstract class ChunkedNlpInferenceResults extends NlpInferenceResults { + + static String TEXT = "text"; + static String INFERENCE = "inference"; + + ChunkedNlpInferenceResults(boolean isTruncated) { + super(isTruncated); + } + + ChunkedNlpInferenceResults(StreamInput in) throws IOException { + super(in); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextEmbeddingResults.java new file mode 100644 index 0000000000000..e47554aebbadf --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextEmbeddingResults.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.results; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class ChunkedTextEmbeddingResults extends ChunkedNlpInferenceResults { + + public record EmbeddingChunk(String matchedText, double[] embedding) implements Writeable, ToXContentObject { + + public EmbeddingChunk(StreamInput in) throws IOException { + this(in.readString(), in.readDoubleArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(matchedText); + out.writeDoubleArray(embedding); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TEXT, matchedText); + builder.field(INFERENCE, embedding); + builder.endObject(); + return builder; + } + + public Map asMap() { + var map = new HashMap(); + map.put(TEXT, matchedText); + map.put(INFERENCE, embedding); + return map; + } + + /** + * It appears the default equals function for a record + * does not call Arrays.equals() for the embedding array. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EmbeddingChunk that = (EmbeddingChunk) o; + return Objects.equals(matchedText, that.matchedText) && Arrays.equals(embedding, that.embedding); + } + + /** + * Use Arrays.hashCode() on the embedding array + */ + @Override + public int hashCode() { + return Objects.hash(matchedText, Arrays.hashCode(embedding)); + } + } + + public static final String NAME = "chunked_text_embedding_result"; + + private final String resultsField; + private final List chunks; + + public ChunkedTextEmbeddingResults(String resultsField, List embeddings, boolean isTruncated) { + super(isTruncated); + this.resultsField = resultsField; + this.chunks = embeddings; + } + + public ChunkedTextEmbeddingResults(StreamInput in) throws IOException { + super(in); + this.resultsField = in.readString(); + this.chunks = in.readCollectionAsList(EmbeddingChunk::new); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public String getResultsField() { + return resultsField; + } + + @Override + public Object predictedValue() { + throw new UnsupportedOperationException("[" + NAME + "] does not support a single predicted value"); + } + + public List getChunks() { + return chunks; + } + + @Override + void doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.startArray(resultsField); + for (var chunk : chunks) { + chunk.toXContent(builder, params); + } + builder.endArray(); + } + + @Override + void doWriteTo(StreamOutput out) throws IOException { + out.writeString(resultsField); + out.writeCollection(chunks); + } + + @Override + void addMapFields(Map map) { + map.put(resultsField, chunks.stream().map(EmbeddingChunk::asMap).collect(Collectors.toList())); + } + + @Override + public boolean equals(Object o) { + + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (super.equals(o) == false) return false; + ChunkedTextEmbeddingResults that = (ChunkedTextEmbeddingResults) o; + return Objects.equals(resultsField, that.resultsField) && Objects.equals(chunks, that.chunks); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), resultsField, chunks); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextExpansionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextExpansionResults.java new file mode 100644 index 0000000000000..f13ba80ce1c2a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextExpansionResults.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.results; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class ChunkedTextExpansionResults extends ChunkedNlpInferenceResults { + public static final String NAME = "chunked_text_expansion_result"; + + public record ChunkedResult(String matchedText, List weightedTokens) + implements + Writeable, + ToXContentObject { + + public ChunkedResult(StreamInput in) throws IOException { + this(in.readString(), in.readCollectionAsList(TextExpansionResults.WeightedToken::new)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(matchedText); + out.writeCollection(weightedTokens); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TEXT, matchedText); + builder.startObject(INFERENCE); + for (var weightedToken : weightedTokens) { + weightedToken.toXContent(builder, params); + } + builder.endObject(); + builder.endObject(); + return builder; + } + + public Map asMap() { + var map = new HashMap(); + map.put(TEXT, matchedText); + map.put( + INFERENCE, + weightedTokens.stream() + .collect(Collectors.toMap(TextExpansionResults.WeightedToken::token, TextExpansionResults.WeightedToken::weight)) + ); + return map; + } + } + + private final String resultsField; + private final List chunks; + + public ChunkedTextExpansionResults(String resultField, List chunks, boolean isTruncated) { + super(isTruncated); + this.resultsField = resultField; + this.chunks = chunks; + } + + public ChunkedTextExpansionResults(StreamInput in) throws IOException { + super(in); + this.resultsField = in.readString(); + this.chunks = in.readCollectionAsList(ChunkedResult::new); + } + + @Override + void doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.startArray(resultsField); + for (var chunk : chunks) { + chunk.toXContent(builder, params); + } + builder.endArray(); + } + + @Override + void doWriteTo(StreamOutput out) throws IOException { + out.writeString(resultsField); + out.writeCollection(chunks); + } + + @Override + void addMapFields(Map map) { + map.put(resultsField, chunks.stream().map(ChunkedResult::asMap).collect(Collectors.toList())); + } + + @Override + public Map asMap(String outputField) { + var map = super.asMap(outputField); + map.put(resultsField, chunks.stream().map(ChunkedResult::asMap).collect(Collectors.toList())); + return map; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (super.equals(o) == false) return false; + ChunkedTextExpansionResults that = (ChunkedTextExpansionResults) o; + return Objects.equals(resultsField, that.resultsField) && Objects.equals(chunks, that.chunks); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), resultsField, chunks); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public String getResultsField() { + return resultsField; + } + + public List getChunks() { + return chunks; + } + + @Override + public Object predictedValue() { + throw new UnsupportedOperationException("[" + NAME + "] does not support a single predicted value"); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java index ab45c2f420bd9..71e67b5c89bfe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java @@ -115,7 +115,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { return builder.build(); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { FillMaskConfig.Builder builder = new FillMaskConfig.Builder(this); - return builder.setTokenization(this.getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings())).build(); + return builder.setTokenization(this.getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings())).build(); } else { throw incompatibleUpdateException(update.getName()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java index b87e7e7edbb71..a5693e1b6f6e7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java @@ -167,7 +167,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { Optional.ofNullable(update.getResultsField()).orElse(resultsField) ); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new NerConfig(this.vocabularyConfig, updatedTokenization, this.classificationLabels, this.resultsField); } else { throw incompatibleUpdateException(update.getName()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java index 0e27fc00b9b70..0c17c6d458dde 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java @@ -129,7 +129,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { update.getResultsField() == null ? resultsField : update.getResultsField() ); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new PassThroughConfig(this.vocabularyConfig, updatedTokenization, this.resultsField); } else { throw incompatibleUpdateException(update.getName()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java index 014cdb1dd891f..134933deab917 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java @@ -200,7 +200,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { Optional.ofNullable(configUpdate.getResultsField()).orElse(resultsField) ); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new QuestionAnsweringConfig( question, numTopClasses, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java index 153879d4f61b4..e37ffcb212810 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java @@ -160,7 +160,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { return builder.build(); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new TextClassificationConfig.Builder(this).setTokenization(updatedTokenization).build(); } else { throw incompatibleUpdateException(update.getName()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java index d043c17535636..57493c7d34d72 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java @@ -46,7 +46,7 @@ private static ConstructingObjectParser createParser( ConstructingObjectParser parser = new ConstructingObjectParser<>( NAME, ignoreUnknownFields, - a -> new TextEmbeddingConfig((VocabularyConfig) a[0], (Tokenization) a[1], (String) a[2], (Integer) a[3]) + a -> TextEmbeddingConfig.create((VocabularyConfig) a[0], (Tokenization) a[1], (String) a[2], (Integer) a[3]) ); parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { if (ignoreUnknownFields == false) { @@ -72,31 +72,42 @@ private static ConstructingObjectParser createParser( private final String resultsField; private final Integer embeddingSize; - public TextEmbeddingConfig( + static TextEmbeddingConfig create( @Nullable VocabularyConfig vocabularyConfig, @Nullable Tokenization tokenization, @Nullable String resultsField, @Nullable Integer embeddingSize ) { - this.vocabularyConfig = Optional.ofNullable(vocabularyConfig) - .orElse(new VocabularyConfig(InferenceIndexConstants.nativeDefinitionStore())); - this.tokenization = tokenization == null ? Tokenization.createDefault() : tokenization; - this.resultsField = resultsField; - if (embeddingSize != null && embeddingSize <= 0) { + var config = new TextEmbeddingConfig( + Optional.ofNullable(vocabularyConfig).orElse(new VocabularyConfig(InferenceIndexConstants.nativeDefinitionStore())), + tokenization == null ? Tokenization.createDefault() : tokenization, + resultsField, + embeddingSize + ); + + if (config.embeddingSize != null && config.embeddingSize <= 0) { throw ExceptionsHelper.badRequestException( "[{}] must be a number greater than 0; configured size [{}]", EMBEDDING_SIZE.getPreferredName(), embeddingSize ); } - this.embeddingSize = embeddingSize; - if (this.tokenization.span != -1) { + if (config.tokenization.span != -1) { throw ExceptionsHelper.badRequestException( "[{}] does not support windowing long text sequences; configured span [{}]", NAME, - this.tokenization.span + config.tokenization.span ); } + + return config; + } + + private TextEmbeddingConfig(VocabularyConfig vocabularyConfig, Tokenization tokenization, String resultsField, Integer embeddingSize) { + this.vocabularyConfig = vocabularyConfig; + this.tokenization = tokenization; + this.resultsField = resultsField; + this.embeddingSize = embeddingSize; } public TextEmbeddingConfig(StreamInput in) throws IOException { @@ -155,7 +166,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { embeddingSize ); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new TextEmbeddingConfig(vocabularyConfig, updatedTokenization, resultsField, embeddingSize); } else { throw incompatibleUpdateException(update.getName()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextExpansionConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextExpansionConfig.java index c4d78c9faf219..f4ac89124cddb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextExpansionConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextExpansionConfig.java @@ -130,7 +130,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { Optional.ofNullable(configUpdate.getResultsField()).orElse(resultsField) ); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new TextExpansionConfig(vocabularyConfig, updatedTokenization, resultsField); } else { throw incompatibleUpdateException(update.getName()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java index bbd819891e217..e06af6e97ceae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java @@ -160,7 +160,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { Optional.ofNullable(configUpdate.getSpanScoreFunction()).orElse(spanScoreFunction) ); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new TextSimilarityConfig(text, vocabularyConfig, updatedTokenization, resultsField, spanScoreFunction); } else { throw incompatibleUpdateException(update.getName()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java index 4f301b48cdacc..4fec726b9fa5d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java @@ -52,7 +52,11 @@ public String toString() { } } - record SpanSettings(@Nullable Integer maxSequenceLength, int span) implements Writeable { + public record SpanSettings(@Nullable Integer maxSequenceLength, int span) implements Writeable { + + public SpanSettings(@Nullable Integer maxSequenceLength) { + this(maxSequenceLength, UNSET_SPAN_VALUE); + } SpanSettings(StreamInput in) throws IOException { this(in.readOptionalVInt(), in.readVInt()); @@ -72,11 +76,11 @@ public void writeTo(StreamOutput out) throws IOException { public static final ParseField TRUNCATE = new ParseField("truncate"); public static final ParseField SPAN = new ParseField("span"); - private static final int DEFAULT_MAX_SEQUENCE_LENGTH = 512; + public static final int DEFAULT_MAX_SEQUENCE_LENGTH = 512; private static final boolean DEFAULT_DO_LOWER_CASE = false; private static final boolean DEFAULT_WITH_SPECIAL_TOKENS = true; private static final Truncate DEFAULT_TRUNCATION = Truncate.FIRST; - private static final int UNSET_SPAN_VALUE = -1; + public static final int UNSET_SPAN_VALUE = -1; static void declareCommonFields(ConstructingObjectParser parser) { parser.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), DO_LOWER_CASE); @@ -141,9 +145,8 @@ public Tokenization(StreamInput in) throws IOException { * @param update The settings to update * @return An updated Tokenization */ - public Tokenization updateSpanSettings(SpanSettings update) { + public Tokenization updateWindowSettings(SpanSettings update) { int maxLength = update.maxSequenceLength() == null ? this.maxSequenceLength : update.maxSequenceLength(); - validateSpanAndMaxSequenceLength(maxLength, span); if (update.maxSequenceLength() != null && update.maxSequenceLength() > this.maxSequenceLength) { throw new ElasticsearchStatusException( "Updated max sequence length [{}] cannot be greater " + "than the model's max sequence length [{}]", @@ -153,7 +156,9 @@ public Tokenization updateSpanSettings(SpanSettings update) { ); } - return buildWindowingTokenization(maxLength, update.span()); + int updatedSpan = update.span() == UNSET_SPAN_VALUE ? this.span : update.span(); + validateSpanAndMaxSequenceLength(maxLength, updatedSpan); + return buildWindowingTokenization(maxLength, updatedSpan); } /** @@ -254,6 +259,10 @@ public int getSpan() { return span; } + public int getMaxSequenceLength() { + return maxSequenceLength; + } + public void validateVocabulary(PutTrainedModelVocabularyAction.Request request) { } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdate.java index 2414fe5776438..974b324fcc80b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdate.java @@ -10,6 +10,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import java.io.IOException; import java.util.Objects; @@ -24,7 +25,11 @@ public class TokenizationConfigUpdate implements InferenceConfigUpdate { private final Tokenization.SpanSettings spanSettings; - public TokenizationConfigUpdate(Tokenization.SpanSettings spanSettings) { + public TokenizationConfigUpdate(@Nullable Integer maxSequenceLength, @Nullable Integer span) { + this(span == null ? new Tokenization.SpanSettings(maxSequenceLength) : new Tokenization.SpanSettings(maxSequenceLength, span)); + } + + private TokenizationConfigUpdate(Tokenization.SpanSettings spanSettings) { this.spanSettings = spanSettings; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java index 4c669f289016a..29d9868bb0de1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java @@ -220,7 +220,7 @@ public InferenceConfig apply(InferenceConfigUpdate update) { Optional.ofNullable(configUpdate.getResultsField()).orElse(resultsField) ); } else if (update instanceof TokenizationConfigUpdate tokenizationUpdate) { - var updatedTokenization = getTokenization().updateSpanSettings(tokenizationUpdate.getSpanSettings()); + var updatedTokenization = getTokenization().updateWindowSettings(tokenizationUpdate.getSpanSettings()); return new ZeroShotClassificationConfig( classificationLabels, vocabularyConfig, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BaseUpdateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BaseUpdateApiKeyRequest.java index 3813e8cb496d6..e5e3e3f2cabac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BaseUpdateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/BaseUpdateApiKeyRequest.java @@ -82,6 +82,10 @@ public ActionRequestValidationException validate() { validationException = RoleDescriptorRequestValidator.validate(roleDescriptor, validationException); } } + if (expiration != null && expiration.nanos() <= 0) { + validationException = addValidationError("API key expiration must be in the future", validationException); + } + return validationException; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java index dd2baca058102..96b4ab3d51bce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java @@ -166,7 +166,9 @@ public class ClusterPrivilegeResolver { private static final Set CROSS_CLUSTER_SEARCH_PATTERN = Set.of( RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, RemoteClusterNodesAction.TYPE.name(), - XPackInfoAction.NAME + XPackInfoAction.NAME, + // esql enrich + "cluster:monitor/xpack/enrich/esql/resolve_policy" ); private static final Set CROSS_CLUSTER_REPLICATION_PATTERN = Set.of( RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index 61018b790c309..204b81914816a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -26,6 +26,7 @@ public final class TransformField { public static final ParseField WAIT_FOR_COMPLETION = new ParseField("wait_for_completion"); public static final ParseField WAIT_FOR_CHECKPOINT = new ParseField("wait_for_checkpoint"); public static final ParseField STATS_FIELD = new ParseField("stats"); + public static final ParseField BASIC_STATS = new ParseField("basic"); public static final ParseField INDEX_DOC_TYPE = new ParseField("doc_type"); public static final ParseField SOURCE = new ParseField("source"); public static final ParseField DESCRIPTION = new ParseField("description"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java index b7259f9bd8d60..0333322d2acc5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.transform.action; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.TaskOperationFailure; @@ -53,12 +54,13 @@ public static final class Request extends BaseTasksRequest { private final String id; private PageParams pageParams = PageParams.defaultParams(); private boolean allowNoMatch = true; + private final boolean basic; public static final int MAX_SIZE_RETURN = 1000; // used internally to expand the queried id expression private List expandedIds; - public Request(String id, @Nullable TimeValue timeout) { + public Request(String id, @Nullable TimeValue timeout, boolean basic) { setTimeout(timeout); if (Strings.isNullOrEmpty(id) || id.equals("*")) { this.id = Metadata.ALL; @@ -66,6 +68,7 @@ public Request(String id, @Nullable TimeValue timeout) { this.id = id; } this.expandedIds = Collections.singletonList(this.id); + this.basic = basic; } public Request(StreamInput in) throws IOException { @@ -74,6 +77,11 @@ public Request(StreamInput in) throws IOException { expandedIds = in.readCollectionAsImmutableList(StreamInput::readString); pageParams = new PageParams(in); allowNoMatch = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_BASIC_STATS)) { + basic = in.readBoolean(); + } else { + basic = false; + } } @Override @@ -111,6 +119,10 @@ public void setAllowNoMatch(boolean allowNoMatch) { this.allowNoMatch = allowNoMatch; } + public boolean isBasic() { + return basic; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -118,6 +130,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(expandedIds); pageParams.writeTo(out); out.writeBoolean(allowNoMatch); + if (out.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_BASIC_STATS)) { + out.writeBoolean(basic); + } } @Override @@ -134,7 +149,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(id, pageParams, allowNoMatch); + return Objects.hash(id, pageParams, allowNoMatch, basic); } @Override @@ -146,7 +161,10 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(id, other.id) && Objects.equals(pageParams, other.pageParams) && allowNoMatch == other.allowNoMatch; + return Objects.equals(id, other.id) + && Objects.equals(pageParams, other.pageParams) + && allowNoMatch == other.allowNoMatch + && basic == other.basic; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java index b9f186ec10833..5e28e74c2e063 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PutTransformAction.java @@ -31,7 +31,20 @@ public class PutTransformAction extends ActionType { public static final PutTransformAction INSTANCE = new PutTransformAction(); public static final String NAME = "cluster:admin/transform/put"; + /** + * Minimum transform frequency used for validation. + * + * Note: Depending on the environment (on-prem or serverless) the minimum frequency used by scheduler can be higher than this constant. + * The actual value used by scheduler is specified by the {@code TransformExtension.getMinFrequency} method. + * + * Example: + * If the user configures transform with frequency=3s but the TransformExtension.getMinFrequency method returns 5s, the validation will + * pass but the scheduler will silently use 5s instead of 3s. + */ private static final TimeValue MIN_FREQUENCY = TimeValue.timeValueSeconds(1); + /** + * Maximum transform frequency used for validation. + */ private static final TimeValue MAX_FREQUENCY = TimeValue.timeValueHours(1); private PutTransformAction() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java index 74b61d24bed41..16e25f031cd55 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -261,24 +260,5 @@ public String toString() { public String value() { return name().toLowerCase(Locale.ROOT); } - - // only used when speaking to nodes < 7.4 (can be removed for 8.0) - public Tuple toComponents() { - - return switch (this) { - case STARTED -> new Tuple<>(TransformTaskState.STARTED, IndexerState.STARTED); - case INDEXING -> new Tuple<>(TransformTaskState.STARTED, IndexerState.INDEXING); - case ABORTING -> new Tuple<>(TransformTaskState.STARTED, IndexerState.ABORTING); - case STOPPING -> - // This one is not deterministic, because an overall state of STOPPING could arise - // from either (STARTED, STOPPED) or (STARTED, STOPPING). However, (STARTED, STOPPED) - // is a very short-lived state so it's reasonable to assume the other, especially - // as this method is only for mixed version cluster compatibility. - new Tuple<>(TransformTaskState.STARTED, IndexerState.STOPPING); - case STOPPED -> new Tuple<>(TransformTaskState.STOPPED, null); - case FAILED -> new Tuple<>(TransformTaskState.FAILED, null); - default -> throw new IllegalStateException("Unexpected state enum value: " + this); - }; - } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 78c62d45177b4..bd267d19398b0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.http.HttpPreRequest; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.index.IndexModule; @@ -239,7 +240,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { List handlers = new ArrayList<>( super.getRestHandlers( @@ -250,7 +252,8 @@ public List getRestHandlers( indexScopedSettings, settingsFilter, indexNameExpressionResolver, - nodesInCluster + nodesInCluster, + clusterSupportsFeature ) ); filterPlugins(ActionPlugin.class).forEach( @@ -263,7 +266,8 @@ public List getRestHandlers( indexScopedSettings, settingsFilter, indexNameExpressionResolver, - nodesInCluster + nodesInCluster, + clusterSupportsFeature ) ) ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java index f97d9e1f21d07..34428c303a076 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlConfigVersionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.utils.MlConfigVersionUtils; import org.hamcrest.Matchers; @@ -149,6 +150,51 @@ public void testGetMinMaxMlConfigVersion() { assertEquals(MlConfigVersion.V_10, MlConfigVersion.getMaxMlConfigVersion(nodes)); } + public void testGetMinMaxMlConfigVersionWhenMlConfigVersionAttrIsMissing() { + Map nodeAttr1 = Map.of(MlConfigVersion.ML_CONFIG_VERSION_NODE_ATTR, MlConfigVersion.V_7_1_0.toString()); + Map nodeAttr2 = Map.of(MlConfigVersion.ML_CONFIG_VERSION_NODE_ATTR, MlConfigVersion.V_8_2_0.toString()); + Map nodeAttr3 = Map.of(); + DiscoveryNodes nodes = DiscoveryNodes.builder() + .add( + DiscoveryNodeUtils.builder("_node_id1") + .name("_node_name1") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300)) + .attributes(nodeAttr1) + .roles(ROLES_WITH_ML) + .version(VersionInformation.inferVersions(Version.fromString("7.2.0"))) + .build() + ) + .add( + DiscoveryNodeUtils.builder("_node_id2") + .name("_node_name2") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9301)) + .attributes(nodeAttr2) + .roles(ROLES_WITH_ML) + .version(VersionInformation.inferVersions(Version.fromString("7.1.0"))) + .build() + ) + .add( + DiscoveryNodeUtils.builder("_node_id3") + .name("_node_name3") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9302)) + .attributes(nodeAttr3) + .roles(ROLES_WITH_ML) + .version( + new VersionInformation( + Version.V_8_11_0, + IndexVersion.getMinimumCompatibleIndexVersion(Version.V_8_11_0.id), + IndexVersion.fromId(Version.V_8_11_0.id) + ) + ) + .build() + ) + .build(); + + assertEquals(MlConfigVersion.V_7_1_0, MlConfigVersion.getMinMlConfigVersion(nodes)); + // _node_name3 is ignored + assertEquals(MlConfigVersion.V_8_2_0, MlConfigVersion.getMaxMlConfigVersion(nodes)); + } + public void testGetMlConfigVersionForNode() { DiscoveryNode node = DiscoveryNodeUtils.builder("_node_id4") .name("_node_name4") diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java index e7d7a7e0926d1..e130951da662f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java @@ -61,6 +61,7 @@ protected InferTrainedModelDeploymentAction.Request createTestInstance() { } request.setHighPriority(randomBoolean()); request.setPrefixType(randomFrom(TrainedModelPrefixStrings.PrefixType.values())); + request.setChunkResults(randomBoolean()); return request; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextEmbeddingResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextEmbeddingResultsTests.java new file mode 100644 index 0000000000000..30ca426c07035 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextEmbeddingResultsTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.results; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD; + +public class ChunkedTextEmbeddingResultsTests extends AbstractWireSerializingTestCase { + + public static ChunkedTextEmbeddingResults createRandomResults() { + var chunks = new ArrayList(); + int columns = randomIntBetween(5, 10); + int numChunks = randomIntBetween(1, 5); + + for (int i = 0; i < numChunks; i++) { + double[] arr = new double[columns]; + for (int j = 0; j < columns; j++) { + arr[j] = randomDouble(); + } + chunks.add(new ChunkedTextEmbeddingResults.EmbeddingChunk(randomAlphaOfLength(6), arr)); + } + + return new ChunkedTextEmbeddingResults(DEFAULT_RESULTS_FIELD, chunks, randomBoolean()); + } + + @Override + protected Writeable.Reader instanceReader() { + return ChunkedTextEmbeddingResults::new; + } + + @Override + protected ChunkedTextEmbeddingResults createTestInstance() { + return createRandomResults(); + } + + @Override + protected ChunkedTextEmbeddingResults mutateInstance(ChunkedTextEmbeddingResults instance) throws IOException { + return switch (randomIntBetween(0, 1)) { + case 0 -> new ChunkedTextEmbeddingResults(instance.getResultsField() + "foo", instance.getChunks(), instance.isTruncated); + case 1 -> new ChunkedTextEmbeddingResults(instance.getResultsField(), instance.getChunks(), instance.isTruncated == false); + default -> throw new IllegalArgumentException("unexpected case"); + }; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextExpansionResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextExpansionResultsTests.java new file mode 100644 index 0000000000000..f29ed8ca6627b --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ChunkedTextExpansionResultsTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.results; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD; + +public class ChunkedTextExpansionResultsTests extends AbstractWireSerializingTestCase { + + public static ChunkedTextExpansionResults createRandomResults() { + var chunks = new ArrayList(); + int numChunks = randomIntBetween(1, 5); + + for (int i = 0; i < numChunks; i++) { + var tokenWeights = new ArrayList(); + int numTokens = randomIntBetween(1, 8); + for (int j = 0; j < numTokens; j++) { + tokenWeights.add(new TextExpansionResults.WeightedToken(Integer.toString(j), (float) randomDoubleBetween(0.0, 5.0, false))); + } + chunks.add(new ChunkedTextExpansionResults.ChunkedResult(randomAlphaOfLength(6), tokenWeights)); + } + + return new ChunkedTextExpansionResults(DEFAULT_RESULTS_FIELD, chunks, randomBoolean()); + } + + @Override + protected Writeable.Reader instanceReader() { + return ChunkedTextExpansionResults::new; + } + + @Override + protected ChunkedTextExpansionResults createTestInstance() { + return createRandomResults(); + } + + @Override + protected ChunkedTextExpansionResults mutateInstance(ChunkedTextExpansionResults instance) throws IOException { + return null; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java index b9cda9a2068ea..6d382bc0a5fe5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java @@ -16,6 +16,8 @@ import java.io.IOException; +import static org.hamcrest.Matchers.containsString; + public class BertTokenizationTests extends AbstractBWCSerializationTestCase { private boolean lenient; @@ -70,6 +72,49 @@ public void testsBuildUpdatedTokenization() { assertEquals(20, update.getSpan()); } + public void testUpdateWindowSettings() { + var tokenization = new BertTokenization(true, true, 100, Tokenization.Truncate.FIRST, -1); + { + var update = tokenization.updateWindowSettings(new Tokenization.SpanSettings((Integer) null)); + // settings not changed + assertEquals(tokenization.getMaxSequenceLength(), update.getMaxSequenceLength()); + assertEquals(tokenization.getSpan(), update.getSpan()); + } + { + var update = tokenization.updateWindowSettings(new Tokenization.SpanSettings(20)); + assertEquals(20, update.getMaxSequenceLength()); + assertEquals(tokenization.getSpan(), update.getSpan()); + } + { + var update = tokenization.updateWindowSettings(new Tokenization.SpanSettings(null, 10)); + assertEquals(tokenization.getMaxSequenceLength(), update.getMaxSequenceLength()); + assertEquals(10, update.getSpan()); + } + { + var update = tokenization.updateWindowSettings(new Tokenization.SpanSettings(20, 10)); + assertEquals(20, update.getMaxSequenceLength()); + assertEquals(10, update.getSpan()); + } + } + + public void testUpdateWindowSettings_InvalidSpan() { + var tokenization = new BertTokenization(true, true, 100, Tokenization.Truncate.FIRST, -1); + var e = expectThrows( + IllegalArgumentException.class, + () -> tokenization.updateWindowSettings(new Tokenization.SpanSettings(32, 64)) + ); + assertThat(e.getMessage(), containsString("[span] provided [64] must not be greater than [max_sequence_length] provided [32]")); + } + + public void testUpdateWindowSettings_InvalidWindowSize() { + var tokenization = new BertTokenization(true, true, 100, Tokenization.Truncate.FIRST, -1); + var e = expectThrows( + IllegalArgumentException.class, + () -> tokenization.updateWindowSettings(new Tokenization.SpanSettings(32, 64)) + ); + assertThat(e.getMessage(), containsString("[span] provided [64] must not be greater than [max_sequence_length] provided [32]")); + } + public static BertTokenization createRandom() { return new BertTokenization( randomBoolean() ? null : randomBoolean(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java index 6d466c31bd56f..4998181173cbb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java @@ -21,14 +21,14 @@ public class TextEmbeddingConfigTests extends InferenceConfigItemTestCase new TextEmbeddingConfig(null, BertTokenizationTests.createRandom(), null, 0) + () -> TextEmbeddingConfig.create(null, BertTokenizationTests.createRandom(), null, 0) ); assertEquals("[embedding_size] must be a number greater than 0; configured size [0]", e.getMessage()); var invalidTokenization = new BertTokenization(true, true, 512, Tokenization.Truncate.NONE, 128); - e = expectThrows(ElasticsearchStatusException.class, () -> new TextEmbeddingConfig(null, invalidTokenization, null, 200)); + e = expectThrows(ElasticsearchStatusException.class, () -> TextEmbeddingConfig.create(null, invalidTokenization, null, 200)); assertEquals("[text_embedding] does not support windowing long text sequences; configured span [128]", e.getMessage()); } public static TextEmbeddingConfig createRandom() { - return new TextEmbeddingConfig( + return TextEmbeddingConfig.create( randomBoolean() ? null : VocabularyConfigTests.createRandom(), randomBoolean() ? null diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java index ecff9c1010c46..2df71b8470948 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java @@ -63,7 +63,7 @@ public void testApply() { assertThat(originalConfig, equalTo(originalConfig.apply(new TextEmbeddingConfigUpdate.Builder().build()))); assertThat( - new TextEmbeddingConfig( + TextEmbeddingConfig.create( originalConfig.getVocabularyConfig(), originalConfig.getTokenization(), "ml-results", @@ -75,7 +75,7 @@ public void testApply() { Tokenization.Truncate truncate = randomFrom(Tokenization.Truncate.values()); Tokenization tokenization = cloneWithNewTruncation(originalConfig.getTokenization(), truncate); assertThat( - new TextEmbeddingConfig( + TextEmbeddingConfig.create( originalConfig.getVocabularyConfig(), tokenization, originalConfig.getResultsField(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdateTests.java index 431dcf6c8c769..90b5c60a01b62 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TokenizationConfigUpdateTests.java @@ -22,7 +22,7 @@ protected Writeable.Reader instanceReader() { protected TokenizationConfigUpdate createTestInstance() { Integer maxSequenceLength = randomBoolean() ? null : randomIntBetween(32, 64); int span = randomIntBetween(8, 16); - return new TokenizationConfigUpdate(new Tokenization.SpanSettings(maxSequenceLength, span)); + return new TokenizationConfigUpdate(maxSequenceLength, span); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java index 7e835dac0df51..82d674fc9b0d6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java @@ -27,7 +27,8 @@ public class GetTransformStatsActionRequestTests extends AbstractWireSerializing protected Request createTestInstance() { return new Request( randomBoolean() ? randomAlphaOfLengthBetween(1, 20) : randomBoolean() ? Metadata.ALL : null, - randomBoolean() ? TimeValue.parseTimeValue(randomTimeValue(), "timeout") : null + randomBoolean() ? TimeValue.parseTimeValue(randomTimeValue(), "timeout") : null, + randomBoolean() ); } @@ -42,7 +43,7 @@ protected Writeable.Reader instanceReader() { } public void testCreateTask() { - Request request = new Request("some-transform", null); + Request request = new Request("some-transform", null, false); Task task = request.createTask(123, "type", "action", TaskId.EMPTY_TASK_ID, Map.of()); assertThat(task, is(instanceOf(CancellableTask.class))); assertThat(task.getDescription(), is(equalTo("get_transform_stats[some-transform]"))); diff --git a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationPlugin.java b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationPlugin.java index 8080761983136..3867e02ac6ca7 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationPlugin.java +++ b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationPlugin.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; @@ -23,6 +24,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.singletonList; @@ -41,7 +43,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Collections.singletonList(new TestDeprecationHeaderRestAction(settings)); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index 329370929ec53..4e2c9da25e78b 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -29,6 +30,7 @@ import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.xpack.deprecation.DeprecationChecks.SKIP_DEPRECATIONS_SETTING; @@ -70,7 +72,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestDeprecationInfoAction(), new RestDeprecationCacheResetAction()); diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 0eb93c59c5b1d..b8443bb2554aa 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -297,8 +297,8 @@ setup: --- "Downsample index": - skip: - version: " - 8.4.99" - reason: "Downsampling GA-ed in 8.7.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -318,28 +318,29 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.0._source.metricset: pod } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 } - - match: { hits.hits.0._source.k8s.pod.scaled-counter: 20.0 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 } + - match: { hits.hits.0._source.k8s.pod.multi-counter: 0 } + - match: { hits.hits.0._source.k8s.pod.scaled-counter: 0.00 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 100 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 102 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 607 } - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } - - match: { hits.hits.0._source.k8s.pod.scaled-gauge.min: 90.0 } - - match: { hits.hits.0._source.k8s.pod.scaled-gauge.max: 100.0 } - - match: { hits.hits.0._source.k8s.pod.scaled-gauge.sum: 190.0 } + - match: { hits.hits.0._source.k8s.pod.scaled-gauge.min: 100.0 } + - match: { hits.hits.0._source.k8s.pod.scaled-gauge.max: 101.0 } + - match: { hits.hits.0._source.k8s.pod.scaled-gauge.sum: 201.0 } - match: { hits.hits.0._source.k8s.pod.scaled-gauge.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 } - - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 } + - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 } + - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434577921 } - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" } - - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 } - - match: { hits.hits.0._source.k8s.pod.tags: ["backend", "prod", "us-west1"] } - - match: { hits.hits.0._source.k8s.pod.values: [1, 1, 3] } - - is_true: hits.hits.0._source.k8s.pod.running + - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.56" } + - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:43:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.number_of_containers: 1 } + - match: { hits.hits.0._source.k8s.pod.tags: ["backend", "test", "us-west2"] } + - match: { hits.hits.0._source.k8s.pod.values: [1, 1, 2] } + - is_false: hits.hits.0._source.k8s.pod.running # Assert rollup index settings - do: @@ -730,8 +731,8 @@ setup: --- "Downsample a downsampled index": - skip: - version: " - 8.6.99" - reason: "Rollup GA-ed in 8.7.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -782,33 +783,33 @@ setup: sort: [ "_tsid", "@timestamp" ] - length: { hits.hits: 3 } - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.0._source._doc_count: 4 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.0._source.metricset: pod } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } - - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 } - - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 } - - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" } - - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 } - - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "prod", "us-west1" ] } - - match: { hits.hits.0._source.k8s.pod.values: [ 1, 1, 3 ] } + - match: { hits.hits.0._source.k8s.pod.multi-counter: 76 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 95.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 110.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 1209.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 12 } + - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 } + - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434595272 } + - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 4 } + - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.120" } + - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:45:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.number_of_containers: 1 } + - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "test", "us-west1" ] } + - match: { hits.hits.0._source.k8s.pod.values: [ 1, 2, 3 ] } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.@timestamp: 2021-04-28T20:00:00.000Z } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.1._source._doc_count: 2 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.2._source.metricset: pod } - - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.2._source._doc_count: 4 } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T20:00:00.000Z } + - match: { hits.hits.2._source._doc_count: 2 } - do: indices.downsample: @@ -869,8 +870,8 @@ setup: --- "Downsample histogram as label": - skip: - version: " - 8.4.99" - reason: "rollup renamed to downsample in 8.5.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -907,65 +908,66 @@ setup: sort: [ "_tsid", "@timestamp" ] - length: { hits.hits: 4 } - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.metricset: pod } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - length: { hits.hits.0._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.0: 8 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.1: 7 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.2: 10 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.3: 12 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.0: 2 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.1: 2 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.2: 8 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.3: 8 } - length: { hits.hits.0._source.k8s.pod.latency.values: 4 } - - match: { hits.hits.0._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.0._source.k8s.pod.latency.values.1: 2.0 } - - match: { hits.hits.0._source.k8s.pod.latency.values.2: 5.0 } - - match: { hits.hits.0._source.k8s.pod.latency.values.3: 10.0 } - - - match: { hits.hits.1._source._doc_count: 2 } - - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z } + - match: { hits.hits.0._source.k8s.pod.latency.values.0: 1.0 } + - match: { hits.hits.0._source.k8s.pod.latency.values.1: 10.0 } + - match: { hits.hits.0._source.k8s.pod.latency.values.2: 100.0 } + - match: { hits.hits.0._source.k8s.pod.latency.values.3: 1000.0 } + + - match: { hits.hits.1._source._doc_count: 1 } + - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.1._source.metricset: pod } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z } - length: { hits.hits.1._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.0: 7 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.1: 15 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.2: 10 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.3: 10 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.0: 4 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.1: 5 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.2: 4 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.3: 13 } - length: { hits.hits.1._source.k8s.pod.latency.values: 4 } - - match: { hits.hits.1._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.1._source.k8s.pod.latency.values.1: 2.0 } - - match: { hits.hits.1._source.k8s.pod.latency.values.2: 5.0 } - - match: { hits.hits.1._source.k8s.pod.latency.values.3: 10.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.0: 1.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.1: 10.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.2: 100.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.3: 1000.0 } - match: { hits.hits.2._source._doc_count: 2 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.2._source.metricset: pod } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z } - length: { hits.hits.2._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.0: 2 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.1: 2 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.2: 8 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.3: 8 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.0: 8 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.1: 7 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.2: 10 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.3: 12 } - length: { hits.hits.2._source.k8s.pod.latency.values: 4 } - match: { hits.hits.2._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.2._source.k8s.pod.latency.values.1: 10.0 } - - match: { hits.hits.2._source.k8s.pod.latency.values.2: 100.0 } - - match: { hits.hits.2._source.k8s.pod.latency.values.3: 1000.0 } + - match: { hits.hits.2._source.k8s.pod.latency.values.1: 2.0 } + - match: { hits.hits.2._source.k8s.pod.latency.values.2: 5.0 } + - match: { hits.hits.2._source.k8s.pod.latency.values.3: 10.0 } - - match: { hits.hits.3._source._doc_count: 1 } - - match: { hits.hits.3._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.3._source._doc_count: 2 } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.3._source.metricset: pod } - match: { hits.hits.3._source.@timestamp: 2021-04-28T19:00:00.000Z } - length: { hits.hits.3._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.0: 4 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.1: 5 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.2: 4 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.3: 13 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.0: 7 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.1: 15 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.2: 10 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.3: 10 } - length: { hits.hits.3._source.k8s.pod.latency.values: 4 } - match: { hits.hits.3._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.3._source.k8s.pod.latency.values.1: 10.0 } - - match: { hits.hits.3._source.k8s.pod.latency.values.2: 100.0 } - - match: { hits.hits.3._source.k8s.pod.latency.values.3: 1000.0 } + - match: { hits.hits.3._source.k8s.pod.latency.values.1: 2.0 } + - match: { hits.hits.3._source.k8s.pod.latency.values.2: 5.0 } + - match: { hits.hits.3._source.k8s.pod.latency.values.3: 10.0 } --- "Downsample date_nanos timestamp field using custom format": @@ -1280,8 +1282,8 @@ setup: --- "Downsample object field": - skip: - version: " - 8.7.99" - reason: "Bug fixed in version 8.8.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -1302,54 +1304,54 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.0._source.metricset: pod } - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.name: "cat" } - - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 } - - match: { hits.hits.0._source.k8s.pod.value.max: 20.0 } - - match: { hits.hits.0._source.k8s.pod.value.sum: 30.0 } - - match: { hits.hits.0._source.k8s.pod.agent.id: "first" } - - match: { hits.hits.0._source.k8s.pod.agent.version: "2.0.4" } + - match: { hits.hits.0._source.k8s.pod.name: "dog" } + - match: { hits.hits.0._source.k8s.pod.value.min: 9.0 } + - match: { hits.hits.0._source.k8s.pod.value.max: 16.0 } + - match: { hits.hits.0._source.k8s.pod.value.sum: 25.0 } + - match: { hits.hits.0._source.k8s.pod.agent.id: "second" } + - match: { hits.hits.0._source.k8s.pod.agent.version: "2.1.7" } - match: { hits.hits.1._source._doc_count: 2 } - - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.@timestamp: "2021-04-28T20:00:00.000Z" } - - match: { hits.hits.1._source.k8s.pod.name: "cat" } - - match: { hits.hits.1._source.k8s.pod.value.min: 12.0 } - - match: { hits.hits.1._source.k8s.pod.value.max: 15.0 } - - match: { hits.hits.1._source.k8s.pod.value.sum: 27.0 } - - match: { hits.hits.1._source.k8s.pod.agent.id: "first" } - - match: { hits.hits.1._source.k8s.pod.agent.version: "2.0.4" } + - match: { hits.hits.1._source.@timestamp: "2021-04-28T19:00:00.000Z" } + - match: { hits.hits.1._source.k8s.pod.name: "dog" } + - match: { hits.hits.1._source.k8s.pod.value.min: 17.0 } + - match: { hits.hits.1._source.k8s.pod.value.max: 25.0 } + - match: { hits.hits.1._source.k8s.pod.value.sum: 42.0 } + - match: { hits.hits.1._source.k8s.pod.agent.id: "second" } + - match: { hits.hits.1._source.k8s.pod.agent.version: "2.1.7" } - match: { hits.hits.2._source._doc_count: 2 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.2._source.metricset: pod } - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.2._source.k8s.pod.name: "dog" } - - match: { hits.hits.2._source.k8s.pod.value.min: 9.0 } - - match: { hits.hits.2._source.k8s.pod.value.max: 16.0 } - - match: { hits.hits.2._source.k8s.pod.value.sum: 25.0 } - - match: { hits.hits.2._source.k8s.pod.agent.id: "second" } - - match: { hits.hits.2._source.k8s.pod.agent.version: "2.1.7" } + - match: { hits.hits.2._source.k8s.pod.name: "cat" } + - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 } + - match: { hits.hits.2._source.k8s.pod.value.max: 20.0 } + - match: { hits.hits.2._source.k8s.pod.value.sum: 30.0 } + - match: { hits.hits.2._source.k8s.pod.agent.id: "first" } + - match: { hits.hits.2._source.k8s.pod.agent.version: "2.0.4" } - match: { hits.hits.3._source._doc_count: 2 } - - match: { hits.hits.3._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.3._source.metricset: pod } - - match: { hits.hits.3._source.@timestamp: "2021-04-28T19:00:00.000Z" } - - match: { hits.hits.3._source.k8s.pod.name: "dog" } - - match: { hits.hits.3._source.k8s.pod.value.min: 17.0 } - - match: { hits.hits.3._source.k8s.pod.value.max: 25.0 } - - match: { hits.hits.3._source.k8s.pod.value.sum: 42.0 } - - match: { hits.hits.3._source.k8s.pod.agent.id: "second" } - - match: { hits.hits.3._source.k8s.pod.agent.version: "2.1.7" } + - match: { hits.hits.3._source.@timestamp: "2021-04-28T20:00:00.000Z" } + - match: { hits.hits.3._source.k8s.pod.name: "cat" } + - match: { hits.hits.3._source.k8s.pod.value.min: 12.0 } + - match: { hits.hits.3._source.k8s.pod.value.max: 15.0 } + - match: { hits.hits.3._source.k8s.pod.value.sum: 27.0 } + - match: { hits.hits.3._source.k8s.pod.agent.id: "first" } + - match: { hits.hits.3._source.k8s.pod.agent.version: "2.0.4" } --- "Downsample empty and missing labels": - skip: - version: " - 8.6.99" - reason: "Downsampling GA-ed in 8.7.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -1369,17 +1371,17 @@ setup: - length: { hits.hits: 3 } - - match: { hits.hits.0._source._doc_count: 4 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.name: "cat" } - - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 } - - match: { hits.hits.0._source.k8s.pod.value.max: 40.0 } - - match: { hits.hits.0._source.k8s.pod.value.sum: 100.0 } - - match: { hits.hits.0._source.k8s.pod.value.value_count: 4 } - - match: { hits.hits.0._source.k8s.pod.label: "abc" } - - match: { hits.hits.0._source.k8s.pod.unmapped: "abc" } + - match: { hits.hits.2._source._doc_count: 4 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.2._source.metricset: pod } + - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.2._source.k8s.pod.name: "cat" } + - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 } + - match: { hits.hits.2._source.k8s.pod.value.max: 40.0 } + - match: { hits.hits.2._source.k8s.pod.value.sum: 100.0 } + - match: { hits.hits.2._source.k8s.pod.value.value_count: 4 } + - match: { hits.hits.2._source.k8s.pod.label: "abc" } + - match: { hits.hits.2._source.k8s.pod.unmapped: "abc" } - match: { hits.hits.1._source._doc_count: 4 } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e9597ab } @@ -1393,24 +1395,24 @@ setup: - match: { hits.hits.1._source.k8s.pod.label: null } - match: { hits.hits.1._source.k8s.pod.unmapped: null } - - match: { hits.hits.2._source._doc_count: 4 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.2._source.metricset: pod } - - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.2._source.k8s.pod.name: "dog" } - - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 } - - match: { hits.hits.2._source.k8s.pod.value.max: 40.0 } - - match: { hits.hits.2._source.k8s.pod.value.sum: 100.0 } - - match: { hits.hits.2._source.k8s.pod.value.value_count: 4 } - - match: { hits.hits.2._source.k8s.pod.label: "xyz" } - - match: { hits.hits.2._source.k8s.pod.unmapped: "xyz" } + - match: { hits.hits.0._source._doc_count: 4 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.metricset: pod } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.name: "dog" } + - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 } + - match: { hits.hits.0._source.k8s.pod.value.max: 40.0 } + - match: { hits.hits.0._source.k8s.pod.value.sum: 100.0 } + - match: { hits.hits.0._source.k8s.pod.value.value_count: 4 } + - match: { hits.hits.0._source.k8s.pod.label: "xyz" } + - match: { hits.hits.0._source.k8s.pod.unmapped: "xyz" } --- "Downsample label with ignore_above": - skip: - version: " - 8.7.99" - reason: "Downsample of time series index without metric allowed from version 8.8.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -1493,37 +1495,37 @@ setup: - match: { hits.hits.0._source._doc_count: 2 } - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.k8s.pod.name: fox } + - match: { hits.hits.0._source.k8s.pod.name: dog } - match: { hits.hits.0._source.k8s.pod.value: 20 } - - match: { hits.hits.0._source.k8s.pod.uid: 7393ef8e-489c-11ee-be56-0242ac120002 } - - match: { hits.hits.0._source.k8s.pod.label: bar } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.k8s.pod.label: foo } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.1._source._doc_count: 2 } - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.k8s.pod.name: cat } + - match: { hits.hits.1._source.k8s.pod.name: fox } - match: { hits.hits.1._source.k8s.pod.value: 20 } - - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - # NOTE: when downsampling a label field we propagate the last (most-recent timestamp-wise) non-null value, - # ignoring/skipping null values. Here the last document has a value that hits ignore_above ("foofoo") and, - # as a result, we propagate the value of the previous document ("foo") - - match: { hits.hits.1._source.k8s.pod.label: foo } + - match: { hits.hits.1._source.k8s.pod.uid: 7393ef8e-489c-11ee-be56-0242ac120002 } + - match: { hits.hits.1._source.k8s.pod.label: bar } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.2._source._doc_count: 2 } - match: { hits.hits.2._source.metricset: pod } - - match: { hits.hits.2._source.k8s.pod.name: cow } + - match: { hits.hits.2._source.k8s.pod.name: cat } - match: { hits.hits.2._source.k8s.pod.value: 20 } - - match: { hits.hits.2._source.k8s.pod.uid: a81ef23a-489c-11ee-be56-0242ac120005 } - - match: { hits.hits.2._source.k8s.pod.label: null } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + # NOTE: when downsampling a label field we propagate the last (most-recent timestamp-wise) non-null value, + # ignoring/skipping null values. Here the last document has a value that hits ignore_above ("foofoo") and, + # as a result, we propagate the value of the previous document ("foo") + - match: { hits.hits.2._source.k8s.pod.label: foo } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.3._source._doc_count: 2 } - match: { hits.hits.3._source.metricset: pod } - - match: { hits.hits.3._source.k8s.pod.name: dog } + - match: { hits.hits.3._source.k8s.pod.name: cow } - match: { hits.hits.3._source.k8s.pod.value: 20 } - - match: { hits.hits.3._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._source.k8s.pod.label: foo } + - match: { hits.hits.3._source.k8s.pod.uid: a81ef23a-489c-11ee-be56-0242ac120005 } + - match: { hits.hits.3._source.k8s.pod.label: null } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:00:00.000Z } - do: @@ -1532,3 +1534,169 @@ setup: - match: { test-downsample-label-ignore-above.mappings.properties.k8s.properties.pod.properties.label.type: keyword } - match: { test-downsample-label-ignore-above.mappings.properties.k8s.properties.pod.properties.label.ignore_above: 3 } + +--- +"Downsample index with empty dimension": + - skip: + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 + + - do: + indices.create: + index: test-empty-dim + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index: + mode: time_series + routing_path: [ k8s.pod.name ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + name: + type: keyword + time_series_dimension: true + empty: + type: keyword + time_series_dimension: true + gauge: + type: long + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test-empty-dim + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "k8s": {"pod": {"name": "cat", "gauge": 10, "empty": "" }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:14.467Z", "k8s": {"pod": {"name": "cat", "gauge": 20 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "k8s": {"pod": {"name": "cat", "gauge": 12, "empty": null }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:34.467Z", "k8s": {"pod": {"name": "cat", "gauge": 18 }}}' + + - do: + indices.put_settings: + index: test-empty-dim + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test-empty-dim + target_index: test-empty-dim-downsample + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: test-empty-dim-downsample + body: + sort: [ "_tsid", "@timestamp" ] + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source._doc_count: 3 } + - match: { hits.hits.0._source.k8s.pod.name: cat } + - match: { hits.hits.0._source.k8s.pod.empty: null } + - match: { hits.hits.1._source._doc_count: 1 } + - match: { hits.hits.1._source.k8s.pod.name: cat } + - match: { hits.hits.1._source.k8s.pod.empty: "" } + +--- +"Downsample index with empty dimension on routing path": + - skip: + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 + + - do: + indices.create: + index: test-empty-dim-routing + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index: + mode: time_series + routing_path: [ k8s.pod.name, k8s.pod.empty ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + name: + type: keyword + time_series_dimension: true + empty: + type: keyword + time_series_dimension: true + gauge: + type: long + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test-empty-dim-routing + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "k8s": {"pod": {"name": "cat", "gauge": 10, "empty": "" }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:14.467Z", "k8s": {"pod": {"name": "cat", "gauge": 20 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "k8s": {"pod": {"name": "cat", "gauge": 12, "empty": null }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:34.467Z", "k8s": {"pod": {"name": "cat", "gauge": 18 }}}' + + - do: + indices.put_settings: + index: test-empty-dim-routing + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test-empty-dim-routing + target_index: test-empty-dim-routing-downsample + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: test-empty-dim-routing-downsample + body: + sort: [ "_tsid", "@timestamp" ] + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source._doc_count: 3 } + - match: { hits.hits.0._source.k8s.pod.name: cat } + - match: { hits.hits.0._source.k8s.pod.empty: null } + - match: { hits.hits.1._source._doc_count: 1 } + - match: { hits.hits.1._source.k8s.pod.name: cat } + - match: { hits.hits.1._source.k8s.pod.empty: "" } diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml index 831ad158deda4..4d64127e2fb88 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml @@ -74,37 +74,37 @@ setup: - match: { hits.hits.0._index: "test-downsample" } - match: { hits.hits.0._source._doc_count: 1 } - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.uid: "001" } - - close_to: { hits.hits.0._source.total_memory_used.min: { value: 106780.0, error: 0.00001 } } - - close_to: { hits.hits.0._source.total_memory_used.max: { value: 106780.0, error: 0.00001 } } - - close_to: { hits.hits.0._source.total_memory_used.sum: { value: 106780.0, error: 0.00001 } } + - match: { hits.hits.0._source.uid: "003" } + - close_to: { hits.hits.0._source.total_memory_used.min: { value: 109009.0, error: 0.00001 } } + - close_to: { hits.hits.0._source.total_memory_used.max: { value: 109009.0, error: 0.00001 } } + - close_to: { hits.hits.0._source.total_memory_used.sum: { value: 109009.0, error: 0.00001 } } - match: { hits.hits.0._source.total_memory_used.value_count: 1 } - match: { hits.hits.1._index: "test-downsample" } - match: { hits.hits.1._source._doc_count: 1 } - match: { hits.hits.1._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.1._source.uid: "002" } - - close_to: { hits.hits.1._source.total_memory_used.min: { value: 110450.0, error: 0.00001 } } - - close_to: { hits.hits.1._source.total_memory_used.max: { value: 110450.0, error: 0.00001 } } - - close_to: { hits.hits.1._source.total_memory_used.sum: { value: 110450.0, error: 0.00001 } } + - match: { hits.hits.1._source.uid: "004" } + - close_to: { hits.hits.1._source.total_memory_used.min: { value: 120770.0, error: 0.00001 } } + - close_to: { hits.hits.1._source.total_memory_used.max: { value: 120770.0, error: 0.00001 } } + - close_to: { hits.hits.1._source.total_memory_used.sum: { value: 120770.0, error: 0.00001 } } - match: { hits.hits.1._source.total_memory_used.value_count: 1 } - match: { hits.hits.2._index: "test-downsample" } - match: { hits.hits.2._source._doc_count: 1 } - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.2._source.uid: "003" } - - close_to: { hits.hits.2._source.total_memory_used.min: { value: 109009.0, error: 0.00001 } } - - close_to: { hits.hits.2._source.total_memory_used.max: { value: 109009.0, error: 0.00001 } } - - close_to: { hits.hits.2._source.total_memory_used.sum: { value: 109009.0, error: 0.00001 } } + - match: { hits.hits.2._source.uid: "002" } + - close_to: { hits.hits.2._source.total_memory_used.min: { value: 110450.0, error: 0.00001 } } + - close_to: { hits.hits.2._source.total_memory_used.max: { value: 110450.0, error: 0.00001 } } + - close_to: { hits.hits.2._source.total_memory_used.sum: { value: 110450.0, error: 0.00001 } } - match: { hits.hits.2._source.total_memory_used.value_count: 1 } - match: { hits.hits.3._index: "test-downsample" } - match: { hits.hits.3._source._doc_count: 1 } - match: { hits.hits.3._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.3._source.uid: "004" } - - close_to: { hits.hits.3._source.total_memory_used.min: { value: 120770.0, error: 0.00001 } } - - close_to: { hits.hits.3._source.total_memory_used.max: { value: 120770.0, error: 0.00001 } } - - close_to: { hits.hits.3._source.total_memory_used.sum: { value: 120770.0, error: 0.00001 } } + - match: { hits.hits.3._source.uid: "001" } + - close_to: { hits.hits.3._source.total_memory_used.min: { value: 106780.0, error: 0.00001 } } + - close_to: { hits.hits.3._source.total_memory_used.max: { value: 106780.0, error: 0.00001 } } + - close_to: { hits.hits.3._source.total_memory_used.sum: { value: 106780.0, error: 0.00001 } } - match: { hits.hits.3._source.total_memory_used.value_count: 1 } # date histogram aggregation with calendar interval on rollup index not supported diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml index 06d74494e89c7..b9dbf621c60dc 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml @@ -1,8 +1,8 @@ --- "Runtime fields accessing metric fields in downsample target index": - skip: - version: " - 8.4.99" - reason: "downsample introduced in 8.5.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 features: close_to - do: @@ -140,31 +140,31 @@ - length: { hits.hits: 4 } - - close_to: { hits.hits.0.fields.received_kb.0: { value: 783343.5488, error: 0.0001 } } - - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1954908.8779, error: 0.0001 } } - - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1958181.5957, error: 0.0001 } } - - close_to: { hits.hits.0.fields.rx_kb.0: { value: 783333.7832, error: 0.0001 } } + - close_to: { hits.hits.0.fields.received_kb.0: { value: 518142.3935, error: 0.0001 } } + - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1400935.4472, error: 0.0001 } } + - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1400955.0009, error: 0.0001 } } + - close_to: { hits.hits.0.fields.rx_kb.0: { value: 518164.1484, error: 0.0001 } } - - close_to: { hits.hits.1.fields.received_kb.0: { value: 783377.7343, error: 0.0001 } } - - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1955339.7343, error: 0.0001 } } - - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1965738.4785, error: 0.0001 } } - - close_to: { hits.hits.1.fields.rx_kb.0: { value: 784849.3369, error: 0.0001 } } + - close_to: { hits.hits.1.fields.received_kb.0: { value: 518186.5039, error: 0.0001 } } + - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1400988.2822, error: 0.0001 } } + - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1400971.9453, error: 0.0001 } } + - close_to: { hits.hits.1.fields.rx_kb.0: { value: 518169.4443, error: 0.0001 } } - - close_to: { hits.hits.2.fields.received_kb.0: { value: 518142.3935, error: 0.0001 } } - - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1400935.4472, error: 0.0001 } } - - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1400955.0009, error: 0.0001 } } - - close_to: { hits.hits.2.fields.rx_kb.0: { value: 518164.1484, error: 0.0001 } } + - close_to: { hits.hits.2.fields.received_kb.0: { value: 783343.5488, error: 0.0001 } } + - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1954908.8779, error: 0.0001 } } + - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1958181.5957, error: 0.0001 } } + - close_to: { hits.hits.2.fields.rx_kb.0: { value: 783333.7832, error: 0.0001 } } - - close_to: { hits.hits.3.fields.received_kb.0: { value: 518186.5039, error: 0.0001 } } - - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1400988.2822, error: 0.0001 } } - - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1400971.9453, error: 0.0001 } } - - close_to: { hits.hits.3.fields.rx_kb.0: { value: 518169.4443, error: 0.0001 } } + - close_to: { hits.hits.3.fields.received_kb.0: { value: 783377.7343, error: 0.0001 } } + - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1955339.7343, error: 0.0001 } } + - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1965738.4785, error: 0.0001 } } + - close_to: { hits.hits.3.fields.rx_kb.0: { value: 784849.3369, error: 0.0001 } } --- "Runtime field accessing dimension fields in downsample target index": - skip: - version: " - 8.4.99" - reason: "downsample introduced in 8.5.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -276,16 +276,16 @@ - length: { hits.hits: 4 } - - match: { hits.hits.0.fields.metricset_tag.0: "pod-AAA" } - - match: { hits.hits.1.fields.metricset_tag.0: "pod-AAB" } - - match: { hits.hits.2.fields.metricset_tag.0: "pod-AAC" } - - match: { hits.hits.3.fields.metricset_tag.0: "pod-AAD" } + - match: { hits.hits.0.fields.metricset_tag.0: "pod-AAD" } + - match: { hits.hits.1.fields.metricset_tag.0: "pod-AAC" } + - match: { hits.hits.2.fields.metricset_tag.0: "pod-AAB" } + - match: { hits.hits.3.fields.metricset_tag.0: "pod-AAA" } --- "Runtime field accessing label fields in downsample target index": - skip: - version: " - 8.4.99" - reason: "downsample introduced in 8.5.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -398,7 +398,7 @@ - length: { hits.hits: 4 } - - match: { hits.hits.0.fields.labels.0: "AAA-10" } - - match: { hits.hits.1.fields.labels.0: "AAB-110" } - - match: { hits.hits.2.fields.labels.0: "AAC-11" } - - match: { hits.hits.3.fields.labels.0: "AAD-111" } + - match: { hits.hits.0.fields.labels.0: "AAC-11" } + - match: { hits.hits.1.fields.labels.0: "AAD-111" } + - match: { hits.hits.2.fields.labels.0: "AAA-10" } + - match: { hits.hits.3.fields.labels.0: "AAB-110" } diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml index 5bc00251c9163..e533db97b3f7e 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml @@ -93,11 +93,13 @@ --- "Downsample datastream with tier preference": - skip: - version: " - 8.4.99" - features: default_shards - reason: "rollup renamed to downsample in 8.5.0, avoid globalTemplateIndexSettings with overlapping index pattern" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 + features: allowed_warnings - do: + allowed_warnings: + - "index template [downsampling-template] has index patterns [test-datastream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [downsampling-template] will take precedence during new index creation" indices.put_index_template: name: downsampling-template body: diff --git a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 2ef436f61a3c9..a1f7ac650141a 100644 --- a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -293,8 +293,8 @@ setup: --- "Downsample index": - skip: - version: " - 8.4.99" - reason: "Downsampling GA-ed in 8.7.0" + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -313,24 +313,24 @@ setup: sort: [ "_tsid", "@timestamp" ] - length: { hits.hits: 4 } - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } - - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 } - - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 } - - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" } - - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 } - - match: { hits.hits.0._source.k8s.pod.tags: ["backend", "prod", "us-west1"] } - - match: { hits.hits.0._source.k8s.pod.values: [1, 1, 3] } - - is_true: hits.hits.0._source.k8s.pod.running + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.metricset: pod } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.multi-counter: 0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 100.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 102.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 607.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } + - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 } + - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434577921 } + - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } + - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.56" } + - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:43:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.number_of_containers: 1 } + - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "test", "us-west2" ] } + - match: { hits.hits.0._source.k8s.pod.values: [ 1, 1, 2 ] } + - is_false: hits.hits.0._source.k8s.pod.running # Assert downsample index settings - do: diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index e0d1fa45a80c3..8d45d66702bd0 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -56,7 +56,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return settings.build(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99520") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105068") @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { final InternalTestCluster cluster = internalCluster(); @@ -132,7 +132,8 @@ public boolean validateClusterForming() { final String targetIndex = "downsample-5m-" + sourceIndex; assertBusy(() -> { try { - GetSettingsResponse getSettingsResponse = client().admin() + GetSettingsResponse getSettingsResponse = cluster.client() + .admin() .indices() .getSettings(new GetSettingsRequest().indices(targetIndex)) .actionGet(); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java new file mode 100644 index 0000000000000..69493e6de442e --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class DimensionFieldProducer extends AbstractDownsampleFieldProducer { + private final Dimension dimension; + + DimensionFieldProducer(final String name, final Dimension dimension) { + super(name); + this.dimension = dimension; + } + + static class Dimension { + private final String name; + private Object value; + private boolean isEmpty; + + Dimension(String name) { + this.name = name; + this.isEmpty = true; + } + + public Object value() { + return value; + } + + public String name() { + return name; + } + + void reset() { + value = null; + isEmpty = true; + } + + void collect(final Object value) { + Objects.requireNonNull(value); + if (isEmpty) { + this.value = value; + this.isEmpty = false; + return; + } + if (value.equals(this.value) == false) { + throw new IllegalArgumentException("Dimension value changed without tsid change [" + value + "] != [" + this.value + "]"); + } + } + } + + @Override + public void reset() { + this.dimension.reset(); + } + + @Override + public boolean isEmpty() { + return this.dimension.isEmpty; + } + + @Override + public void collect(FormattedDocValues docValues, int docId) throws IOException { + if (docValues.advanceExact(docId) == false) { + return; + } + int docValueCount = docValues.docValueCount(); + for (int i = 0; i < docValueCount; i++) { + this.dimension.collect(docValues.nextValue()); + } + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(this.dimension.name, this.dimension.value()); + } + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java new file mode 100644 index 0000000000000..c6ef43cfdacfa --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class DimensionFieldValueFetcher extends FieldValueFetcher { + + private final DimensionFieldProducer dimensionFieldProducer = createFieldProducer(); + + protected DimensionFieldValueFetcher(final MappedFieldType fieldType, final IndexFieldData fieldData) { + super(fieldType.name(), fieldType, fieldData); + } + + private DimensionFieldProducer createFieldProducer() { + final String filedName = fieldType.name(); + return new DimensionFieldProducer(filedName, new DimensionFieldProducer.Dimension(filedName)); + } + + @Override + public AbstractDownsampleFieldProducer fieldProducer() { + return this.dimensionFieldProducer; + } + + /** + * Retrieve field value fetchers for a list of dimensions. + */ + static List create(final SearchExecutionContext context, final String[] dimensions) { + List fetchers = new ArrayList<>(); + for (String dimension : dimensions) { + MappedFieldType fieldType = context.getFieldType(dimension); + assert fieldType != null : "Unknown dimension field type for dimension field: [" + dimension + "]"; + + if (context.fieldExistsInIndex(dimension)) { + final IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); + final String fieldName = context.isMultiField(dimension) + ? fieldType.name().substring(0, fieldType.name().lastIndexOf('.')) + : fieldType.name(); + fetchers.add(new DimensionFieldValueFetcher(fieldType, fieldData)); + } + } + return Collections.unmodifiableList(fetchers); + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java index 260782a3eb0f3..6e2a966a566a6 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.persistent.PersistentTasksExecutor; @@ -39,6 +40,7 @@ import org.elasticsearch.xpack.core.downsample.DownsampleShardTask; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class Downsample extends Plugin implements ActionPlugin, PersistentTaskPlugin { @@ -81,7 +83,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestDownsampleAction()); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index f74bd299916c1..844c644ee9ea6 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -94,6 +94,7 @@ class DownsampleShardIndexer { private final List fieldValueFetchers; private final DownsampleShardTask task; private final DownsampleShardPersistentTaskState state; + private final String[] dimensions; private volatile boolean abort = false; ByteSizeValue downsampleBulkSize = DOWNSAMPLE_BULK_SIZE; ByteSizeValue downsampleMaxBytesInFlight = DOWNSAMPLE_MAX_BYTES_IN_FLIGHT; @@ -107,6 +108,7 @@ class DownsampleShardIndexer { final DownsampleConfig config, final String[] metrics, final String[] labels, + final String[] dimensions, final DownsampleShardPersistentTaskState state ) { this.task = task; @@ -125,13 +127,15 @@ class DownsampleShardIndexer { null, Collections.emptyMap() ); + this.dimensions = dimensions; this.timestampField = (DateFieldMapper.DateFieldType) searchExecutionContext.getFieldType(config.getTimestampField()); this.timestampFormat = timestampField.docValueFormat(null, null); this.rounding = config.createRounding(); - List fetchers = new ArrayList<>(metrics.length + labels.length); + List fetchers = new ArrayList<>(metrics.length + labels.length + dimensions.length); fetchers.addAll(FieldValueFetcher.create(searchExecutionContext, metrics)); fetchers.addAll(FieldValueFetcher.create(searchExecutionContext, labels)); + fetchers.addAll(DimensionFieldValueFetcher.create(searchExecutionContext, dimensions)); this.fieldValueFetchers = Collections.unmodifiableList(fetchers); toClose = null; } finally { @@ -155,7 +159,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept BulkProcessor2 bulkProcessor = createBulkProcessor(); try (searcher; bulkProcessor) { final TimeSeriesIndexSearcher timeSeriesSearcher = new TimeSeriesIndexSearcher(searcher, List.of(this::checkCancelled)); - TimeSeriesBucketCollector bucketCollector = new TimeSeriesBucketCollector(bulkProcessor); + TimeSeriesBucketCollector bucketCollector = new TimeSeriesBucketCollector(bulkProcessor, this.dimensions); bucketCollector.preCollection(); timeSeriesSearcher.search(initialStateQuery, bucketCollector); } @@ -332,12 +336,12 @@ private class TimeSeriesBucketCollector extends BucketCollector { long lastTimestamp = Long.MAX_VALUE; long lastHistoTimestamp = Long.MAX_VALUE; - TimeSeriesBucketCollector(BulkProcessor2 bulkProcessor) { + TimeSeriesBucketCollector(BulkProcessor2 bulkProcessor, String[] dimensions) { this.bulkProcessor = bulkProcessor; AbstractDownsampleFieldProducer[] fieldProducers = fieldValueFetchers.stream() .map(FieldValueFetcher::fieldProducer) .toArray(AbstractDownsampleFieldProducer[]::new); - this.downsampleBucketBuilder = new DownsampleBucketBuilder(fieldProducers); + this.downsampleBucketBuilder = new DownsampleBucketBuilder(fieldProducers, dimensions); } @Override @@ -358,12 +362,12 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag @Override public void collect(int docId, long owningBucketOrd) throws IOException { task.addNumReceived(1); - final BytesRef tsid = aggCtx.getTsid(); - assert tsid != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; - final int tsidOrd = aggCtx.getTsidOrd(); + final BytesRef tsidHash = aggCtx.getTsidHash(); + assert tsidHash != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; + final int tsidHashOrd = aggCtx.getTsidHashOrd(); final long timestamp = timestampField.resolution().roundDownToMillis(aggCtx.getTimestamp()); - boolean tsidChanged = tsidOrd != downsampleBucketBuilder.tsidOrd(); + boolean tsidChanged = tsidHashOrd != downsampleBucketBuilder.tsidOrd(); if (tsidChanged || timestamp < lastHistoTimestamp) { lastHistoTimestamp = Math.max( rounding.round(timestamp), @@ -377,7 +381,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { logger.trace( "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> downsample bucket ts: [{}]", docId, - DocValueFormat.TIME_SERIES_ID.format(tsid), + DocValueFormat.TIME_SERIES_ID.format(tsidHash), timestampFormat.format(timestamp), timestampFormat.format(lastHistoTimestamp) ); @@ -389,13 +393,13 @@ public void collect(int docId, long owningBucketOrd) throws IOException { * - @timestamp must be sorted in descending order within the same _tsid */ BytesRef lastTsid = downsampleBucketBuilder.tsid(); - assert lastTsid == null || lastTsid.compareTo(tsid) <= 0 + assert lastTsid == null || lastTsid.compareTo(tsidHash) <= 0 : "_tsid is not sorted in ascending order: [" + DocValueFormat.TIME_SERIES_ID.format(lastTsid) + "] -> [" - + DocValueFormat.TIME_SERIES_ID.format(tsid) + + DocValueFormat.TIME_SERIES_ID.format(tsidHash) + "]"; - assert tsid.equals(lastTsid) == false || lastTimestamp >= timestamp + assert tsidHash.equals(lastTsid) == false || lastTimestamp >= timestamp : "@timestamp is not sorted in descending order: [" + timestampFormat.format(lastTimestamp) + "] -> [" @@ -412,7 +416,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { // Create new downsample bucket if (tsidChanged) { - downsampleBucketBuilder.resetTsid(tsid, tsidOrd, lastHistoTimestamp); + downsampleBucketBuilder.resetTsid(tsidHash, tsidHashOrd, lastHistoTimestamp); } else { downsampleBucketBuilder.resetTimestamp(lastHistoTimestamp); } @@ -482,9 +486,11 @@ private class DownsampleBucketBuilder { private int docCount; private final AbstractDownsampleFieldProducer[] fieldProducers; private final DownsampleFieldSerializer[] groupedProducers; + private final String[] dimensions; - DownsampleBucketBuilder(AbstractDownsampleFieldProducer[] fieldProducers) { + DownsampleBucketBuilder(AbstractDownsampleFieldProducer[] fieldProducers, String[] dimensions) { this.fieldProducers = fieldProducers; + this.dimensions = dimensions; /* * The downsample field producers for aggregate_metric_double all share the same name (this is * the name they will be serialized in the target index). We group all field producers by @@ -545,12 +551,6 @@ public XContentBuilder buildDownsampleDocument() throws IOException { } builder.field(timestampField.name(), timestampFormat.format(timestamp)); builder.field(DocCountFieldMapper.NAME, docCount); - // Extract dimension values from _tsid field, so we avoid loading them from doc_values - Map dimensions = (Map) DocValueFormat.TIME_SERIES_ID.format(tsid); - for (Map.Entry e : dimensions.entrySet()) { - assert e.getValue() != null; - builder.field((String) e.getKey(), e.getValue()); - } // Serialize fields for (DownsampleFieldSerializer fieldProducer : groupedProducers) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java index f500ce986f6dd..6fa09ef2175c4 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java @@ -201,6 +201,7 @@ protected void doRun() throws Exception { params.downsampleConfig(), params.metrics(), params.labels(), + params.dimensions(), initialState ); downsampleShardIndexer.execute(); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java index 813dcc8c8d5a4..4ccc913b974d6 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java @@ -32,7 +32,8 @@ public record DownsampleShardTaskParams( long indexEndTimeMillis, ShardId shardId, String[] metrics, - String[] labels + String[] labels, + String[] dimensions ) implements PersistentTaskParams { public static final String NAME = DownsampleShardTask.TASK_NAME; @@ -43,6 +44,7 @@ public record DownsampleShardTaskParams( private static final ParseField SHARD_ID = new ParseField("shard_id"); private static final ParseField METRICS = new ParseField("metrics"); private static final ParseField LABELS = new ParseField("labels"); + private static final ParseField DIMENSIONS = new ParseField("dimensions"); public static final ObjectParser PARSER = new ObjectParser<>(NAME); static { @@ -57,6 +59,7 @@ public record DownsampleShardTaskParams( PARSER.declareString(DownsampleShardTaskParams.Builder::shardId, SHARD_ID); PARSER.declareStringArray(DownsampleShardTaskParams.Builder::metrics, METRICS); PARSER.declareStringArray(DownsampleShardTaskParams.Builder::labels, LABELS); + PARSER.declareStringArray(DownsampleShardTaskParams.Builder::dimensions, DIMENSIONS); } DownsampleShardTaskParams(final StreamInput in) throws IOException { @@ -67,7 +70,8 @@ public record DownsampleShardTaskParams( in.readVLong(), new ShardId(in), in.readStringArray(), - in.readStringArray() + in.readStringArray(), + in.readOptionalStringArray() ); } @@ -81,6 +85,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SHARD_ID.getPreferredName(), shardId); builder.array(METRICS.getPreferredName(), metrics); builder.array(LABELS.getPreferredName(), labels); + builder.array(DIMENSIONS.getPreferredName(), dimensions); return builder.endObject(); } @@ -103,6 +108,7 @@ public void writeTo(StreamOutput out) throws IOException { shardId.writeTo(out); out.writeStringArray(metrics); out.writeStringArray(labels); + out.writeOptionalStringArray(dimensions); } public static DownsampleShardTaskParams fromXContent(XContentParser parser) throws IOException { @@ -123,7 +129,8 @@ public boolean equals(Object o) { && Objects.equals(shardId.id(), that.shardId.id()) && Objects.equals(shardId.getIndexName(), that.shardId.getIndexName()) && Arrays.equals(metrics, that.metrics) - && Arrays.equals(labels, that.labels); + && Arrays.equals(labels, that.labels) + && Arrays.equals(dimensions, that.dimensions); } @Override @@ -138,6 +145,7 @@ public int hashCode() { ); result = 31 * result + Arrays.hashCode(metrics); result = 31 * result + Arrays.hashCode(labels); + result = 31 * result + Arrays.hashCode(dimensions); return result; } @@ -149,6 +157,7 @@ public static class Builder { ShardId shardId; String[] metrics; String[] labels; + String[] dimensions; public Builder downsampleConfig(final DownsampleConfig downsampleConfig) { this.downsampleConfig = downsampleConfig; @@ -185,6 +194,11 @@ public Builder labels(final List labels) { return this; } + public Builder dimensions(final List dimensions) { + this.dimensions = dimensions.toArray(String[]::new); + return this; + } + public DownsampleShardTaskParams build() { return new DownsampleShardTaskParams( downsampleConfig, @@ -193,7 +207,8 @@ public DownsampleShardTaskParams build() { indexEndTimeMillis, shardId, metrics, - labels + labels, + dimensions ); } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java index 2788932a228a8..74375bbe27939 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java @@ -37,7 +37,7 @@ protected FieldValueFetcher(String name, MappedFieldType fieldType, IndexFieldDa this.name = name; this.fieldType = fieldType; this.fieldData = fieldData; - this.fieldProducer = createieldProducer(); + this.fieldProducer = createFieldProducer(); } public String name() { @@ -53,7 +53,7 @@ public AbstractDownsampleFieldProducer fieldProducer() { return fieldProducer; } - private AbstractDownsampleFieldProducer createieldProducer() { + private AbstractDownsampleFieldProducer createFieldProducer() { if (fieldType.getMetricType() != null) { return switch (fieldType.getMetricType()) { case GAUGE -> new MetricFieldProducer.GaugeMetricFieldProducer(name()); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index f3bb43b9a3f38..c4b48bbb016ef 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -336,7 +336,8 @@ protected void masterOperation( downsampleIndexName, parentTask, metricFields, - labelFields + labelFields, + dimensionFields ); } else { delegate.onFailure(new ElasticsearchException("Failed to create downsample index [" + downsampleIndexName + "]")); @@ -350,7 +351,8 @@ protected void masterOperation( downsampleIndexName, parentTask, metricFields, - labelFields + labelFields, + dimensionFields ); } else { delegate.onFailure(e); @@ -368,7 +370,8 @@ private void performShardDownsampling( String downsampleIndexName, TaskId parentTask, List metricFields, - List labelFields + List labelFields, + List dimensionFields ) { final int numberOfShards = sourceIndexMetadata.getNumberOfShards(); final Index sourceIndex = sourceIndexMetadata.getIndex(); @@ -388,6 +391,7 @@ private void performShardDownsampling( downsampleIndexName, metricFields, labelFields, + dimensionFields, shardId ); Predicate> predicate = runningTask -> { @@ -496,6 +500,7 @@ private static DownsampleShardTaskParams createPersistentTaskParams( final String targetIndexName, final List metricFields, final List labelFields, + final List dimensionFields, final ShardId shardId ) { return new DownsampleShardTaskParams( @@ -505,7 +510,8 @@ private static DownsampleShardTaskParams createPersistentTaskParams( parseTimestamp(sourceIndexMetadata, IndexSettings.TIME_SERIES_END_TIME), shardId, metricFields.toArray(new String[0]), - labelFields.toArray(new String[0]) + labelFields.toArray(new String[0]), + dimensionFields.toArray(new String[0]) ); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java index a7c34cacae5be..24d1df638f80b 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java @@ -144,6 +144,7 @@ protected DownsampleIndexerAction.ShardDownsampleResponse shardOperation( request.getRollupConfig(), request.getMetricFields(), request.getLabelFields(), + request.getDimensionFields(), new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); return indexer.execute(); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 28eb9ae66a4e0..a7b36bbd7dc9b 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -628,6 +628,7 @@ public void testCancelDownsampleIndexer() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); @@ -676,6 +677,7 @@ public void testDownsampleBulkFailed() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); @@ -742,6 +744,7 @@ public void testTooManyBytesInFlight() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); /* @@ -793,6 +796,7 @@ public void testDownsampleStats() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); @@ -849,6 +853,7 @@ public void testResumeDownsample() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState( DownsampleShardIndexerStatus.STARTED, new BytesRef( @@ -923,39 +928,57 @@ public void testResumeDownsamplePartial() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1 }, new DownsampleShardPersistentTaskState( DownsampleShardIndexerStatus.STARTED, + // NOTE: there is just one dimension with two possible values, this needs to be one of the two possible tsid values. new BytesRef( new byte[] { - 0x01, - 0x0C, - 0x64, - 0x69, - 0x6d, - 0x65, - 0x6E, + 0x24, + 0x42, + (byte) 0xe4, + (byte) 0x9f, + (byte) 0xe2, + (byte) 0xde, + (byte) 0xbb, + (byte) 0xf8, + (byte) 0xfc, + 0x7d, + 0x1a, + (byte) 0xb1, + 0x27, + (byte) 0x85, + (byte) 0xc2, + (byte) 0x8e, + 0x3a, + (byte) 0xae, + 0x38, + 0x6c, + (byte) 0xf6, + (byte) 0xae, + 0x0f, + 0x4f, + 0x44, + (byte) 0xf1, 0x73, - 0x69, - 0x6F, - 0x6E, - 0x5F, - 0x6B, - 0x77, - 0x73, - 0x04, - 0x64, - 0x69, - 0x6D, - 0x32 } + 0x02, + (byte) 0x90, + 0x1d, + 0x79, + (byte) 0xf8, + 0x0d, + (byte) 0xc2, + 0x7e, + (byte) 0x91, + 0x15 } ) ) ); final DownsampleIndexerAction.ShardDownsampleResponse response2 = indexer.execute(); long dim2DocCount = SearchResponseUtils.getTotalHitsValue( - client().prepareSearch(sourceIndex).setQuery(new TermQueryBuilder(FIELD_DIMENSION_1, "dim2")).setSize(10_000) + client().prepareSearch(sourceIndex).setQuery(new TermQueryBuilder(FIELD_DIMENSION_1, "dim1")).setSize(10_000) ); - assertDownsampleIndexer(indexService, shardNum, task, response2, dim2DocCount); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index e9a075227107c..868ec49ff1d97 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.ingest.Processor; import org.elasticsearch.license.XPackLicenseState; @@ -61,6 +62,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.ENRICH_INDEX_PATTERN; @@ -167,7 +169,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of( new RestGetEnrichPolicyAction(), diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/300_connector_put.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/300_connector_put.yml index c7bc5f48a3d89..467443235d061 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/300_connector_put.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/300_connector_put.yml @@ -115,3 +115,23 @@ setup: is_native: false service_type: super-connector + + +--- +'Create Connector - Index name used by another connector': + - do: + connector.put: + connector_id: test-connector-1 + body: + index_name: search-test + + - match: { result: 'created' } + + + - do: + catch: "bad_request" + connector.put: + connector_id: test-connector-2 + body: + index_name: search-test + diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/305_connector_post.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/305_connector_post.yml index 9b7432adf290d..ff465df72b272 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/305_connector_post.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/305_connector_post.yml @@ -88,3 +88,18 @@ setup: is_native: false service_type: super-connector +--- +'Create Connector - Index name used by another connector': + - do: + connector.post: + body: + index_name: search-test + + - set: { id: id } + - match: { id: $id } + + - do: + catch: "bad_request" + connector.post: + body: + index_name: search-test diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/310_connector_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/310_connector_list.yml index 52cfcdee0bb85..7aa49297902d5 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/310_connector_list.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/310_connector_list.yml @@ -9,7 +9,7 @@ setup: connector_id: connector-a body: index_name: search-1-test - name: my-connector + name: my-connector-1 language: pl is_native: false service_type: super-connector @@ -18,7 +18,7 @@ setup: connector_id: connector-c body: index_name: search-3-test - name: my-connector + name: my-connector-3 language: nl is_native: false service_type: super-connector @@ -27,7 +27,7 @@ setup: connector_id: connector-b body: index_name: search-2-test - name: my-connector + name: my-connector-2 language: en is_native: true service_type: super-connector @@ -106,3 +106,61 @@ setup: - match: { count: 0 } + +--- +"List Connector - filter by index names": + - do: + connector.list: + index_name: search-1-test + + - match: { count: 1 } + - match: { results.0.index_name: "search-1-test" } + + - do: + connector.list: + index_name: search-1-test,search-2-test + + - match: { count: 2 } + - match: { results.0.index_name: "search-1-test" } + - match: { results.1.index_name: "search-2-test" } + + +--- +"List Connector - filter by index names, illegal name": + - do: + catch: "bad_request" + connector.list: + index_name: ~.!$$#index-name$$$ + + +--- +"List Connector - filter by connector names": + - do: + connector.list: + connector_name: my-connector-1 + + - match: { count: 1 } + - match: { results.0.name: "my-connector-1" } + + - do: + connector.list: + connector_name: my-connector-1,my-connector-2 + + - match: { count: 2 } + - match: { results.0.name: "my-connector-1" } + - match: { results.1.name: "my-connector-2" } + + +--- +"List Connector - filter by index name and name": + - do: + connector.list: + connector_name: my-connector-1,my-connector-2 + index_name: search-2-test + + - match: { count: 1 } + - match: { results.0.index_name: "search-2-test" } + - match: { results.0.name: "my-connector-2" } + + + diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml index 5a7ab14dc6386..aeac8202a950b 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/335_connector_update_configuration.yml @@ -185,7 +185,6 @@ setup: - field: some_field value: 31 display: numeric - label: Very important field --- "Update Connector Configuration - Unknown field type": @@ -240,3 +239,26 @@ setup: - constraint: 0 type: unknown_constraint value: 123 + +--- +"Update Connector Configuration - Crawler configuration": + - do: + connector.update_configuration: + connector_id: test-connector + body: + configuration: + nextSyncConfig: + label: nextSyncConfig + value: + max_crawl_depth: 3 + sitemap_discovery_disabled: false + seed_urls: + - https://elastic.co/ + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { configuration.nextSyncConfig.value.max_crawl_depth: 3 } diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/338_connector_update_index_name.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/338_connector_update_index_name.yml new file mode 100644 index 0000000000000..1ed7297df346e --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/338_connector_update_index_name.yml @@ -0,0 +1,110 @@ +setup: + - skip: + version: " - 8.12.99" + reason: Introduced in 8.13.0 + + - do: + connector.put: + connector_id: test-connector + body: + index_name: search-1-test + name: my-connector + language: pl + is_native: false + service_type: super-connector + +--- +"Update Connector Index Name": + - do: + connector.update_index_name: + connector_id: test-connector + body: + index_name: search-2-test + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { index_name: search-2-test } + - match: { status: created } + +--- +"Update Connector Index Name - 404 when connector doesn't exist": + - do: + catch: "missing" + connector.update_index_name: + connector_id: test-non-existent-connector + body: + index_name: search-2-test + +--- +"Update Connector Index Name - 400 status code when connector_id is empty": + - do: + catch: "bad_request" + connector.update_index_name: + connector_id: "" + body: + index_name: search-2-test + +--- +"Update Connector Index Name - 400 status code when payload is not string": + - do: + catch: "bad_request" + connector.update_index_name: + connector_id: test-connector + body: + index_name: + field_1: test + field_2: something + + +--- +"Update Connector Index Name - 400 status code when invalid index name": + - do: + catch: "bad_request" + connector.update_index_name: + connector_id: test-connector + body: + index_name: _invalid-index-name + +--- +"Update Connector Index Name - Index name used by another connector": + - do: + connector.put: + connector_id: test-connector-2 + body: + index_name: search-2-test + name: my-connector + language: pl + is_native: false + service_type: super-connector + + - match: { result: created } + + - do: + catch: "bad_request" + connector.update_index_name: + connector_id: test-connector + body: + index_name: search-2-test + +--- +"Update Connector Index Name - Index name is the same": + + - do: + connector.update_index_name: + connector_id: test-connector + body: + index_name: search-1-test + + + - match: { result: noop } + + - do: + connector.get: + connector_id: test-connector + + - match: { index_name: search-1-test } diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/340_connector_update_status.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/340_connector_update_status.yml new file mode 100644 index 0000000000000..8463a19919dd1 --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/340_connector_update_status.yml @@ -0,0 +1,94 @@ +setup: + - skip: + version: " - 8.12.99" + reason: Introduced in 8.13.0 + + - do: + connector.put: + connector_id: test-connector + body: + index_name: search-1-test + name: my-connector + language: pl + is_native: false + service_type: super-connector + +--- +"Update Connector Status": + - do: + connector.update_status: + connector_id: test-connector + body: + status: needs_configuration + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { status: needs_configuration } + + - do: + connector.update_status: + connector_id: test-connector + body: + status: configured + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { status: configured } + +--- +"Update Connector Status - 404 when connector doesn't exist": + - do: + catch: "missing" + connector.update_status: + connector_id: test-non-existent-connector + body: + status: needs_configuration + +--- +"Update Connector Status - 400 status code when connector_id is empty": + - do: + catch: "bad_request" + connector.update_status: + connector_id: "" + body: + status: needs_configuration + +--- +"Update Connector Status - 400 status code when payload is not string": + - do: + catch: "bad_request" + connector.update_status: + connector_id: test-connector + body: + status: + field_1: test + field_2: something + +--- +"Update Connector Status - 400 status code when invalid status": + - do: + catch: "bad_request" + connector.update_status: + connector_id: test-connector + body: + status: _invalid-status + + +--- +"Update Connector Status - 400 status code when invalid status transition": + - do: + catch: "bad_request" + connector.update_status: + connector_id: test-connector + body: + status: connected # created -> connected transition is invalid + + - match: { error.reason: "Invalid transition attempt from [created] to [connected]. Such a status transition is not supported by the Connector Protocol." } diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/341_connector_update_api_key_id.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/341_connector_update_api_key_id.yml new file mode 100644 index 0000000000000..3d82c53acae50 --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/341_connector_update_api_key_id.yml @@ -0,0 +1,110 @@ +setup: + - skip: + version: " - 8.12.99" + reason: Introduced in 8.13.0 + + - do: + connector.put: + connector_id: test-connector + body: + index_name: search-1-test + name: my-connector + language: pl + is_native: false + service_type: super-connector + +--- +"Update Connector Api Key Id": + - do: + connector.update_api_key_id: + connector_id: test-connector + body: + api_key_id: test-api-key-id + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { api_key_id: test-api-key-id } + +--- +"Update Connector Api Key Secret Id": + - do: + connector.update_api_key_id: + connector_id: test-connector + body: + api_key_secret_id: test-api-key-secret-id + + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { api_key_secret_id: test-api-key-secret-id } + +--- +"Update Connector Api Key Id and Api Key Secret Id": + - do: + connector.update_api_key_id: + connector_id: test-connector + body: + api_key_id: test-api-key-id + api_key_secret_id: test-api-key-secret-id + + - match: { result: updated } + + - do: + connector.get: + connector_id: test-connector + + - match: { api_key_id: test-api-key-id } + - match: { api_key_secret_id: test-api-key-secret-id } + +--- +"Update Connector Api Key Id - 404 when connector doesn't exist": + - do: + catch: "missing" + connector.update_api_key_id: + connector_id: test-non-existent-connector + body: + api_key_id: test-api-key-id + api_key_secret_id: test-api-key-secret-id + +--- +"Update Connector Api Key Id - 400 status code when connector_id is empty": + - do: + catch: "bad_request" + connector.update_api_key_id: + connector_id: "" + body: + api_key_id: test-api-key-id + api_key_secret_id: test-api-key-secret-id + +--- +"Update Connector Api Key Id - 400 status code when both values are null": + - do: + catch: "bad_request" + connector.update_api_key_id: + connector_id: test-connector + body: + api_key_id: null + api_key_secret_id: null + + - match: { error.reason: "Validation Failed: 1: [api_key_id] and [api_key_secret_id] cannot both be [null]. Please provide a value for at least one of them.;" } + +--- +"Update Connector Api Key Id - 400 status code when payload is not string": + - do: + catch: "bad_request" + connector.update_api_key_id: + connector_id: test-connector + body: + api_key_id: + field_1: test + field_2: something + api_key_secret_id: test-api-key-secret-id diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index 3933e7923d6b9..b10d1f9e582a0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.logging.LogManager; @@ -53,9 +54,11 @@ import org.elasticsearch.xpack.application.connector.action.RestListConnectorAction; import org.elasticsearch.xpack.application.connector.action.RestPostConnectorAction; import org.elasticsearch.xpack.application.connector.action.RestPutConnectorAction; +import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFilteringAction; +import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorIndexNameAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorLastSyncStatsAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorNameAction; @@ -63,14 +66,17 @@ import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorPipelineAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorSchedulingAction; import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorServiceTypeAction; +import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorStatusAction; import org.elasticsearch.xpack.application.connector.action.TransportDeleteConnectorAction; import org.elasticsearch.xpack.application.connector.action.TransportGetConnectorAction; import org.elasticsearch.xpack.application.connector.action.TransportListConnectorAction; import org.elasticsearch.xpack.application.connector.action.TransportPostConnectorAction; import org.elasticsearch.xpack.application.connector.action.TransportPutConnectorAction; +import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFilteringAction; +import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorIndexNameAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorLastSyncStatsAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorNameAction; @@ -78,9 +84,12 @@ import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorPipelineAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorSchedulingAction; import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorServiceTypeAction; +import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorStatusAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorIndexNameAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSyncStatsAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorNameAction; @@ -88,6 +97,7 @@ import org.elasticsearch.xpack.application.connector.action.UpdateConnectorPipelineAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorSchedulingAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorServiceTypeAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorStatusAction; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsFeature; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsIndexService; import org.elasticsearch.xpack.application.connector.secrets.action.DeleteConnectorSecretAction; @@ -167,6 +177,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.singletonList; @@ -244,9 +255,11 @@ protected XPackLicenseState getLicenseState() { new ActionHandler<>(ListConnectorAction.INSTANCE, TransportListConnectorAction.class), new ActionHandler<>(PostConnectorAction.INSTANCE, TransportPostConnectorAction.class), new ActionHandler<>(PutConnectorAction.INSTANCE, TransportPutConnectorAction.class), + new ActionHandler<>(UpdateConnectorApiKeyIdAction.INSTANCE, TransportUpdateConnectorApiKeyIdAction.class), new ActionHandler<>(UpdateConnectorConfigurationAction.INSTANCE, TransportUpdateConnectorConfigurationAction.class), new ActionHandler<>(UpdateConnectorErrorAction.INSTANCE, TransportUpdateConnectorErrorAction.class), new ActionHandler<>(UpdateConnectorFilteringAction.INSTANCE, TransportUpdateConnectorFilteringAction.class), + new ActionHandler<>(UpdateConnectorIndexNameAction.INSTANCE, TransportUpdateConnectorIndexNameAction.class), new ActionHandler<>(UpdateConnectorLastSeenAction.INSTANCE, TransportUpdateConnectorLastSeenAction.class), new ActionHandler<>(UpdateConnectorLastSyncStatsAction.INSTANCE, TransportUpdateConnectorLastSyncStatsAction.class), new ActionHandler<>(UpdateConnectorNameAction.INSTANCE, TransportUpdateConnectorNameAction.class), @@ -254,6 +267,7 @@ protected XPackLicenseState getLicenseState() { new ActionHandler<>(UpdateConnectorPipelineAction.INSTANCE, TransportUpdateConnectorPipelineAction.class), new ActionHandler<>(UpdateConnectorSchedulingAction.INSTANCE, TransportUpdateConnectorSchedulingAction.class), new ActionHandler<>(UpdateConnectorServiceTypeAction.INSTANCE, TransportUpdateConnectorServiceTypeAction.class), + new ActionHandler<>(UpdateConnectorStatusAction.INSTANCE, TransportUpdateConnectorStatusAction.class), // SyncJob API new ActionHandler<>(GetConnectorSyncJobAction.INSTANCE, TransportGetConnectorSyncJobAction.class), @@ -293,7 +307,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (enabled == false) { @@ -334,9 +349,11 @@ public List getRestHandlers( new RestListConnectorAction(), new RestPostConnectorAction(), new RestPutConnectorAction(), + new RestUpdateConnectorApiKeyIdAction(), new RestUpdateConnectorConfigurationAction(), new RestUpdateConnectorErrorAction(), new RestUpdateConnectorFilteringAction(), + new RestUpdateConnectorIndexNameAction(), new RestUpdateConnectorLastSeenAction(), new RestUpdateConnectorLastSyncStatsAction(), new RestUpdateConnectorNameAction(), @@ -344,6 +361,7 @@ public List getRestHandlers( new RestUpdateConnectorPipelineAction(), new RestUpdateConnectorSchedulingAction(), new RestUpdateConnectorServiceTypeAction(), + new RestUpdateConnectorStatusAction(), // SyncJob API new RestGetConnectorSyncJobAction(), diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java index b7ddf560247ed..5bae203175d36 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -42,6 +42,7 @@ *
    *
  • A doc _id of the connector document.
    • *
  • API key for authenticating with Elasticsearch, ensuring secure access.
    • + *
  • Connector Secret ID for API key, allowing Connectors to access the API key.
    • *
  • A configuration mapping which holds specific settings and parameters for the connector's operation.
    • *
  • A {@link ConnectorCustomSchedule} object that defines custom scheduling.
    • *
  • A description providing an overview or purpose of the connector.
    • @@ -71,6 +72,8 @@ public class Connector implements NamedWriteable, ToXContentObject { private final String connectorId; @Nullable private final String apiKeyId; + @Nullable + private final String apiKeySecretId; private final Map configuration; private final Map customScheduling; @Nullable @@ -108,6 +111,7 @@ public class Connector implements NamedWriteable, ToXContentObject { * * @param connectorId Unique identifier for the connector. Used when building get/list response. Equals to doc _id. * @param apiKeyId API key ID used for authentication/authorization against ES. + * @param apiKeySecretId Connector Secret document ID for API key. * @param configuration Configuration settings for the connector. * @param customScheduling Custom scheduling settings for the connector. * @param description Description of the connector. @@ -131,6 +135,7 @@ public class Connector implements NamedWriteable, ToXContentObject { private Connector( String connectorId, String apiKeyId, + String apiKeySecretId, Map configuration, Map customScheduling, String description, @@ -153,6 +158,7 @@ private Connector( ) { this.connectorId = connectorId; this.apiKeyId = apiKeyId; + this.apiKeySecretId = apiKeySecretId; this.configuration = configuration; this.customScheduling = customScheduling; this.description = description; @@ -177,6 +183,7 @@ private Connector( public Connector(StreamInput in) throws IOException { this.connectorId = in.readOptionalString(); this.apiKeyId = in.readOptionalString(); + this.apiKeySecretId = in.readOptionalString(); this.configuration = in.readMap(ConnectorConfiguration::new); this.customScheduling = in.readMap(ConnectorCustomSchedule::new); this.description = in.readOptionalString(); @@ -199,7 +206,8 @@ public Connector(StreamInput in) throws IOException { } public static final ParseField ID_FIELD = new ParseField("id"); - static final ParseField API_KEY_ID_FIELD = new ParseField("api_key_id"); + public static final ParseField API_KEY_ID_FIELD = new ParseField("api_key_id"); + public static final ParseField API_KEY_SECRET_ID_FIELD = new ParseField("api_key_secret_id"); public static final ParseField CONFIGURATION_FIELD = new ParseField("configuration"); static final ParseField CUSTOM_SCHEDULING_FIELD = new ParseField("custom_scheduling"); public static final ParseField DESCRIPTION_FIELD = new ParseField("description"); @@ -214,7 +222,7 @@ public Connector(StreamInput in) throws IOException { public static final ParseField PIPELINE_FIELD = new ParseField("pipeline"); public static final ParseField SCHEDULING_FIELD = new ParseField("scheduling"); public static final ParseField SERVICE_TYPE_FIELD = new ParseField("service_type"); - static final ParseField STATUS_FIELD = new ParseField("status"); + public static final ParseField STATUS_FIELD = new ParseField("status"); static final ParseField SYNC_CURSOR_FIELD = new ParseField("sync_cursor"); static final ParseField SYNC_NOW_FIELD = new ParseField("sync_now"); @@ -226,6 +234,7 @@ public Connector(StreamInput in) throws IOException { int i = 0; return new Builder().setConnectorId(docId) .setApiKeyId((String) args[i++]) + .setApiKeySecretId((String) args[i++]) .setConfiguration((Map) args[i++]) .setCustomScheduling((Map) args[i++]) .setDescription((String) args[i++]) @@ -262,6 +271,7 @@ public Connector(StreamInput in) throws IOException { static { PARSER.declareStringOrNull(optionalConstructorArg(), API_KEY_ID_FIELD); + PARSER.declareStringOrNull(optionalConstructorArg(), API_KEY_SECRET_ID_FIELD); PARSER.declareObject( optionalConstructorArg(), (p, c) -> p.map(HashMap::new, ConnectorConfiguration::fromXContent), @@ -363,6 +373,7 @@ public void toInnerXContent(XContentBuilder builder, Params params) throws IOExc builder.field(ID_FIELD.getPreferredName(), connectorId); } builder.field(API_KEY_ID_FIELD.getPreferredName(), apiKeyId); + builder.field(API_KEY_SECRET_ID_FIELD.getPreferredName(), apiKeySecretId); builder.xContentValuesMap(CONFIGURATION_FIELD.getPreferredName(), configuration); builder.xContentValuesMap(CUSTOM_SCHEDULING_FIELD.getPreferredName(), customScheduling); builder.field(DESCRIPTION_FIELD.getPreferredName(), description); @@ -397,6 +408,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(connectorId); out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeySecretId); out.writeMap(configuration, StreamOutput::writeWriteable); out.writeMap(customScheduling, StreamOutput::writeWriteable); out.writeOptionalString(description); @@ -426,6 +438,10 @@ public String getApiKeyId() { return apiKeyId; } + public String getApiKeySecretId() { + return apiKeySecretId; + } + public Map getConfiguration() { return configuration; } @@ -511,6 +527,7 @@ public boolean equals(Object o) { && syncNow == connector.syncNow && Objects.equals(connectorId, connector.connectorId) && Objects.equals(apiKeyId, connector.apiKeyId) + && Objects.equals(apiKeySecretId, connector.apiKeySecretId) && Objects.equals(configuration, connector.configuration) && Objects.equals(customScheduling, connector.customScheduling) && Objects.equals(description, connector.description) @@ -535,6 +552,7 @@ public int hashCode() { return Objects.hash( connectorId, apiKeyId, + apiKeySecretId, configuration, customScheduling, description, @@ -566,6 +584,7 @@ public static class Builder { private String connectorId; private String apiKeyId; + private String apiKeySecretId; private Map configuration = Collections.emptyMap(); private Map customScheduling = Collections.emptyMap(); private String description; @@ -596,6 +615,11 @@ public Builder setApiKeyId(String apiKeyId) { return this; } + public Builder setApiKeySecretId(String apiKeySecretId) { + this.apiKeySecretId = apiKeySecretId; + return this; + } + public Builder setConfiguration(Map configuration) { this.configuration = configuration; return this; @@ -686,8 +710,8 @@ public Builder setSyncCursor(Object syncCursor) { return this; } - public Builder setSyncNow(boolean syncNow) { - this.syncNow = syncNow; + public Builder setSyncNow(Boolean syncNow) { + this.syncNow = Objects.requireNonNullElse(syncNow, false); return this; } @@ -695,6 +719,7 @@ public Connector build() { return new Connector( connectorId, apiKeyId, + apiKeySecretId, configuration, customScheduling, description, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java index 8ed7c417a1af1..7d7c7b5fa61f9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorConfiguration.java @@ -162,8 +162,8 @@ public ConnectorConfiguration(StreamInput in) throws IOException { .setOptions((List) args[i++]) .setOrder((Integer) args[i++]) .setPlaceholder((String) args[i++]) - .setRequired((boolean) args[i++]) - .setSensitive((boolean) args[i++]) + .setRequired((Boolean) args[i++]) + .setSensitive((Boolean) args[i++]) .setTooltip((String) args[i++]) .setType((ConfigurationFieldType) args[i++]) .setUiRestrictions((List) args[i++]) @@ -187,40 +187,42 @@ public ConnectorConfiguration(StreamInput in) throws IOException { } throw new XContentParseException("Unsupported token [" + p.currentToken() + "]"); }, DEFAULT_VALUE_FIELD, ObjectParser.ValueType.VALUE); - PARSER.declareObjectArray(constructorArg(), (p, c) -> ConfigurationDependency.fromXContent(p), DEPENDS_ON_FIELD); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ConfigurationDependency.fromXContent(p), DEPENDS_ON_FIELD); PARSER.declareField( - constructorArg(), + optionalConstructorArg(), (p, c) -> ConfigurationDisplayType.displayType(p.text()), DISPLAY_FIELD, - ObjectParser.ValueType.STRING + ObjectParser.ValueType.STRING_OR_NULL ); PARSER.declareString(constructorArg(), LABEL_FIELD); - PARSER.declareObjectArray(constructorArg(), (p, c) -> ConfigurationSelectOption.fromXContent(p), OPTIONS_FIELD); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ConfigurationSelectOption.fromXContent(p), OPTIONS_FIELD); PARSER.declareInt(optionalConstructorArg(), ORDER_FIELD); - PARSER.declareString(optionalConstructorArg(), PLACEHOLDER_FIELD); - PARSER.declareBoolean(constructorArg(), REQUIRED_FIELD); - PARSER.declareBoolean(constructorArg(), SENSITIVE_FIELD); + PARSER.declareStringOrNull(optionalConstructorArg(), PLACEHOLDER_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), REQUIRED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), SENSITIVE_FIELD); PARSER.declareStringOrNull(optionalConstructorArg(), TOOLTIP_FIELD); PARSER.declareField( - constructorArg(), - (p, c) -> ConfigurationFieldType.fieldType(p.text()), + optionalConstructorArg(), + (p, c) -> p.currentToken() == XContentParser.Token.VALUE_NULL ? null : ConfigurationFieldType.fieldType(p.text()), TYPE_FIELD, - ObjectParser.ValueType.STRING + ObjectParser.ValueType.STRING_OR_NULL ); - PARSER.declareStringArray(constructorArg(), UI_RESTRICTIONS_FIELD); - PARSER.declareObjectArray(constructorArg(), (p, c) -> ConfigurationValidation.fromXContent(p), VALIDATIONS_FIELD); - PARSER.declareField(constructorArg(), (p, c) -> { + PARSER.declareStringArray(optionalConstructorArg(), UI_RESTRICTIONS_FIELD); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ConfigurationValidation.fromXContent(p), VALIDATIONS_FIELD); + PARSER.declareField(optionalConstructorArg(), (p, c) -> { if (p.currentToken() == XContentParser.Token.VALUE_STRING) { return p.text(); } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { return p.numberValue(); } else if (p.currentToken() == XContentParser.Token.VALUE_BOOLEAN) { return p.booleanValue(); + } else if (p.currentToken() == XContentParser.Token.START_OBJECT) { + return p.map(); } else if (p.currentToken() == XContentParser.Token.VALUE_NULL) { return null; } throw new XContentParseException("Unsupported token [" + p.currentToken() + "]"); - }, VALUE_FIELD, ObjectParser.ValueType.VALUE); + }, VALUE_FIELD, ObjectParser.ValueType.VALUE_OBJECT_ARRAY); } @Override @@ -231,10 +233,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CATEGORY_FIELD.getPreferredName(), category); } builder.field(DEFAULT_VALUE_FIELD.getPreferredName(), defaultValue); - builder.xContentList(DEPENDS_ON_FIELD.getPreferredName(), dependsOn); - builder.field(DISPLAY_FIELD.getPreferredName(), display.toString()); + if (dependsOn != null) { + builder.xContentList(DEPENDS_ON_FIELD.getPreferredName(), dependsOn); + } + if (display != null) { + builder.field(DISPLAY_FIELD.getPreferredName(), display.toString()); + } builder.field(LABEL_FIELD.getPreferredName(), label); - builder.xContentList(OPTIONS_FIELD.getPreferredName(), options); + if (options != null) { + builder.xContentList(OPTIONS_FIELD.getPreferredName(), options); + } if (order != null) { builder.field(ORDER_FIELD.getPreferredName(), order); } @@ -243,10 +251,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(REQUIRED_FIELD.getPreferredName(), required); builder.field(SENSITIVE_FIELD.getPreferredName(), sensitive); - builder.field(TOOLTIP_FIELD.getPreferredName(), tooltip); - builder.field(TYPE_FIELD.getPreferredName(), type.toString()); - builder.stringListField(UI_RESTRICTIONS_FIELD.getPreferredName(), uiRestrictions); - builder.xContentList(VALIDATIONS_FIELD.getPreferredName(), validations); + if (tooltip != null) { + builder.field(TOOLTIP_FIELD.getPreferredName(), tooltip); + } + if (type != null) { + builder.field(TYPE_FIELD.getPreferredName(), type.toString()); + } + if (uiRestrictions != null) { + builder.stringListField(UI_RESTRICTIONS_FIELD.getPreferredName(), uiRestrictions); + } + if (validations != null) { + builder.xContentList(VALIDATIONS_FIELD.getPreferredName(), validations); + } builder.field(VALUE_FIELD.getPreferredName(), value); } builder.endObject(); @@ -385,13 +401,13 @@ public Builder setPlaceholder(String placeholder) { return this; } - public Builder setRequired(boolean required) { - this.required = required; + public Builder setRequired(Boolean required) { + this.required = Objects.requireNonNullElse(required, false); return this; } - public Builder setSensitive(boolean sensitive) { - this.sensitive = sensitive; + public Builder setSensitive(Boolean sensitive) { + this.sensitive = Objects.requireNonNullElse(sensitive, false); return this; } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index cf6c3190a37b4..b321a497ab58d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.application.connector; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; @@ -25,7 +26,13 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchHit; @@ -34,15 +41,18 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.application.connector.action.PostConnectorAction; import org.elasticsearch.xpack.application.connector.action.PutConnectorAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorIndexNameAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSyncStatsAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorNameAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorNativeAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorPipelineAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorSchedulingAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorServiceTypeAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorStatusAction; import java.time.Instant; import java.util.Arrays; @@ -80,6 +90,9 @@ public ConnectorIndexService(Client client) { */ public void createConnectorWithDocId(PutConnectorAction.Request request, ActionListener listener) { + String indexName = request.getIndexName(); + String connectorId = request.getConnectorId(); + Connector connector = createConnectorWithDefaultValues( request.getDescription(), request.getIndexName(), @@ -90,11 +103,26 @@ public void createConnectorWithDocId(PutConnectorAction.Request request, ActionL ); try { - final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .id(request.getConnectorId()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - clientWithOrigin.index(indexRequest, listener); + isDataIndexNameAlreadyInUse(indexName, connectorId, listener.delegateFailure((l, isIndexNameInUse) -> { + if (isIndexNameInUse) { + l.onFailure( + new ElasticsearchStatusException( + "Index name [" + indexName + "] is used by another connector.", + RestStatus.BAD_REQUEST + ) + ); + return; + } + try { + final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); + clientWithOrigin.index(indexRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + })); } catch (Exception e) { listener.onFailure(e); } @@ -111,9 +139,11 @@ public void createConnectorWithAutoGeneratedId( ActionListener listener ) { + String indexName = request.getIndexName(); + Connector connector = createConnectorWithDefaultValues( request.getDescription(), - request.getIndexName(), + indexName, request.getIsNative(), request.getLanguage(), request.getName(), @@ -121,14 +151,31 @@ public void createConnectorWithAutoGeneratedId( ); try { - final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); + isDataIndexNameAlreadyInUse(indexName, null, listener.delegateFailure((l, isIndexNameInUse) -> { + if (isIndexNameInUse) { + l.onFailure( + new ElasticsearchStatusException( + "Index name [" + indexName + "] is used by another connector.", + RestStatus.BAD_REQUEST + ) + ); + return; + } + try { + final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - clientWithOrigin.index( - indexRequest, - listener.delegateFailureAndWrap((l, indexResponse) -> l.onResponse(new PostConnectorAction.Response(indexResponse.getId()))) - ); + clientWithOrigin.index( + indexRequest, + listener.delegateFailureAndWrap( + (ll, indexResponse) -> ll.onResponse(new PostConnectorAction.Response(indexResponse.getId())) + ) + ); + } catch (Exception e) { + listener.onFailure(e); + } + })); } catch (Exception e) { listener.onFailure(e); } @@ -235,13 +282,21 @@ public void deleteConnector(String connectorId, ActionListener l * * @param from From index to start the search from. * @param size The maximum number of {@link Connector}s to return. + * @param indexNames A list of index names to filter the connectors. + * @param connectorNames A list of connector names to further filter the search results. * @param listener The action listener to invoke on response/failure. */ - public void listConnectors(int from, int size, ActionListener listener) { + public void listConnectors( + int from, + int size, + List indexNames, + List connectorNames, + ActionListener listener + ) { try { final SearchSourceBuilder source = new SearchSourceBuilder().from(from) .size(size) - .query(new MatchAllQueryBuilder()) + .query(buildListQuery(indexNames, connectorNames)) .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); @@ -269,6 +324,33 @@ public void onFailure(Exception e) { } } + /** + * Constructs a query for filtering instances of {@link Connector} based on index and/or connector names. + * Returns a {@link MatchAllQueryBuilder} if both parameters are empty or null, + * otherwise constructs a boolean query to filter by the provided lists. + * + * @param indexNames List of index names to filter by, or null/empty for no index name filtering. + * @param connectorNames List of connector names to filter by, or null/empty for no name filtering. + * @return A {@link QueryBuilder} tailored to the specified filters. + */ + private QueryBuilder buildListQuery(List indexNames, List connectorNames) { + boolean filterByIndexNames = indexNames != null && indexNames.isEmpty() == false; + boolean filterByConnectorNames = indexNames != null && connectorNames.isEmpty() == false; + boolean usesFilter = filterByIndexNames || filterByConnectorNames; + + BoolQueryBuilder boolFilterQueryBuilder = new BoolQueryBuilder(); + + if (usesFilter) { + if (filterByIndexNames) { + boolFilterQueryBuilder.must().add(new TermsQueryBuilder(Connector.INDEX_NAME_FIELD.getPreferredName(), indexNames)); + } + if (filterByConnectorNames) { + boolFilterQueryBuilder.must().add(new TermsQueryBuilder(Connector.NAME_FIELD.getPreferredName(), connectorNames)); + } + } + return usesFilter ? boolFilterQueryBuilder : new MatchAllQueryBuilder(); + } + /** * Updates the {@link ConnectorConfiguration} property of a {@link Connector}. * The update process is non-additive; it completely replaces all existing configuration fields with the new configuration mapping, @@ -543,6 +625,53 @@ public void updateConnectorPipeline(UpdateConnectorPipelineAction.Request reques } } + /** + * Updates the index name property of a {@link Connector}. + * + * @param request The request for updating the connector's index name. + * @param listener The listener for handling responses, including successful updates or errors. + */ + public void updateConnectorIndexName(UpdateConnectorIndexNameAction.Request request, ActionListener listener) { + try { + String connectorId = request.getConnectorId(); + String indexName = request.getIndexName(); + + isDataIndexNameAlreadyInUse(indexName, connectorId, listener.delegateFailure((l, isIndexNameInUse) -> { + + if (isIndexNameInUse) { + l.onFailure( + new ElasticsearchStatusException( + "Index name [" + indexName + "] is used by another connector.", + RestStatus.BAD_REQUEST + ) + ); + return; + } + + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(Map.of(Connector.INDEX_NAME_FIELD.getPreferredName(), request.getIndexName())) + + ); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorId)); + return; + } + ll.onResponse(updateResponse); + }) + ); + })); + + } catch (Exception e) { + listener.onFailure(e); + } + } + /** * Updates the {@link ConnectorScheduling} property of a {@link Connector}. * @@ -584,9 +713,7 @@ public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request String connectorId = request.getConnectorId(); getConnector(connectorId, listener.delegateFailure((l, connector) -> { - ConnectorStatus prevStatus = ConnectorStatus.connectorStatus( - (String) connector.getResultMap().get(Connector.STATUS_FIELD.getPreferredName()) - ); + ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); ConnectorStatus newStatus = prevStatus == ConnectorStatus.CREATED ? ConnectorStatus.CREATED : ConnectorStatus.NEEDS_CONFIGURATION; @@ -621,6 +748,80 @@ public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request } } + /** + * Updates the {@link ConnectorStatus} property of a {@link Connector}. + * + * @param request The request for updating the connector's status. + * @param listener The listener for handling responses, including successful updates or errors. + */ + public void updateConnectorStatus(UpdateConnectorStatusAction.Request request, ActionListener listener) { + try { + String connectorId = request.getConnectorId(); + ConnectorStatus newStatus = request.getStatus(); + getConnector(connectorId, listener.delegateFailure((l, connector) -> { + + ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); + + try { + ConnectorStateMachine.assertValidStateTransition(prevStatus, newStatus); + } catch (ConnectorInvalidStatusTransitionException e) { + l.onFailure(new ElasticsearchStatusException(e.getMessage(), RestStatus.BAD_REQUEST, e)); + return; + } + + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(Map.of(Connector.STATUS_FIELD.getPreferredName(), request.getStatus())) + ); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + updateListener.onFailure(new ResourceNotFoundException(connectorId)); + return; + } + updateListener.onResponse(updateResponse); + }) + ); + })); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void updateConnectorApiKeyIdOrApiKeySecretId( + UpdateConnectorApiKeyIdAction.Request request, + ActionListener listener + ) { + try { + String connectorId = request.getConnectorId(); + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) + ); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorId)); + return; + } + l.onResponse(updateResponse); + }) + ); + } catch (Exception e) { + listener.onFailure(e); + } + } + + private ConnectorStatus getConnectorStatusFromSearchResult(ConnectorSearchResult searchResult) { + return ConnectorStatus.connectorStatus((String) searchResult.getResultMap().get(Connector.STATUS_FIELD.getPreferredName())); + } + private static ConnectorIndexService.ConnectorResult mapSearchResponseToConnectorList(SearchResponse response) { final List connectorResults = Arrays.stream(response.getHits().getHits()) .map(ConnectorIndexService::hitToConnector) @@ -638,6 +839,49 @@ private static ConnectorSearchResult hitToConnector(SearchHit searchHit) { .build(); } + /** + * This method determines if any documents in the connector index have the same index name as the one specified, + * excluding the document with the given _id if it is provided. + * + * @param indexName The name of the index to check for existence in the connector index. + * @param connectorId The ID of the {@link Connector} to exclude from the search. Can be null if no document should be excluded. + * @param listener The listener for handling boolean responses and errors. + */ + private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, ActionListener listener) { + try { + BoolQueryBuilder boolFilterQueryBuilder = new BoolQueryBuilder(); + + boolFilterQueryBuilder.must().add(new TermQueryBuilder(Connector.INDEX_NAME_FIELD.getPreferredName(), indexName)); + + // If we know the connector _id, exclude this from search query + if (connectorId != null) { + boolFilterQueryBuilder.mustNot(new IdsQueryBuilder().addIds(connectorId)); + } + + final SearchSourceBuilder searchSource = new SearchSourceBuilder().query(boolFilterQueryBuilder); + + final SearchRequest searchRequest = new SearchRequest(CONNECTOR_INDEX_NAME).source(searchSource); + clientWithOrigin.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + boolean indexNameIsInUse = searchResponse.getHits().getTotalHits().value > 0L; + listener.onResponse(indexNameIsInUse); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + listener.onResponse(false); + return; + } + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + public record ConnectorResult(List connectors, long totalResults) {} /** diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorInvalidStatusTransitionException.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorInvalidStatusTransitionException.java new file mode 100644 index 0000000000000..dd863d07a9e90 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorInvalidStatusTransitionException.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +public class ConnectorInvalidStatusTransitionException extends Exception { + + /** + * Constructs an ConnectorInvalidStatusTransitionException exception with a detailed message. + * + * @param current The current state of the connector. + * @param next The attempted next state of the connector. + */ + public ConnectorInvalidStatusTransitionException(ConnectorStatus current, ConnectorStatus next) { + super( + "Invalid transition attempt from [" + + current + + "] to [" + + next + + "]. Such a status transition is not supported by the Connector Protocol." + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java index 21bfdbc06ec3c..8e1c4d95d8527 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachine.java @@ -23,7 +23,7 @@ public class ConnectorStateMachine { ConnectorStatus.CREATED, EnumSet.of(ConnectorStatus.NEEDS_CONFIGURATION, ConnectorStatus.ERROR), ConnectorStatus.NEEDS_CONFIGURATION, - EnumSet.of(ConnectorStatus.CONFIGURED), + EnumSet.of(ConnectorStatus.CONFIGURED, ConnectorStatus.ERROR), ConnectorStatus.CONFIGURED, EnumSet.of(ConnectorStatus.NEEDS_CONFIGURATION, ConnectorStatus.CONNECTED, ConnectorStatus.ERROR), ConnectorStatus.CONNECTED, @@ -39,6 +39,16 @@ public class ConnectorStateMachine { * @param next The proposed next state of the connector. */ public static boolean isValidTransition(ConnectorStatus current, ConnectorStatus next) { - return VALID_TRANSITIONS.getOrDefault(current, Collections.emptySet()).contains(next); + return validNextStates(current).contains(next); + } + + public static void assertValidStateTransition(ConnectorStatus current, ConnectorStatus next) + throws ConnectorInvalidStatusTransitionException { + if (isValidTransition(current, next)) return; + throw new ConnectorInvalidStatusTransitionException(current, next); + } + + public static Set validNextStates(ConnectorStatus current) { + return VALID_TRANSITIONS.getOrDefault(current, Collections.emptySet()); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index b4a3a2c0d3632..13a588fdd6314 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -11,8 +11,10 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -26,7 +28,9 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public class ListConnectorAction { @@ -38,54 +42,88 @@ private ListConnectorAction() {/* no instances */} public static class Request extends ActionRequest implements ToXContentObject { private final PageParams pageParams; + private final List indexNames; + private final List connectorNames; private static final ParseField PAGE_PARAMS_FIELD = new ParseField("pageParams"); + private static final ParseField INDEX_NAMES_FIELD = new ParseField("index_names"); + private static final ParseField NAMES_FIELD = new ParseField("names"); public Request(StreamInput in) throws IOException { super(in); this.pageParams = new PageParams(in); + this.indexNames = in.readOptionalStringCollectionAsList(); + this.connectorNames = in.readOptionalStringCollectionAsList(); } - public Request(PageParams pageParams) { + public Request(PageParams pageParams, List indexNames, List connectorNames) { this.pageParams = pageParams; + this.indexNames = indexNames; + this.connectorNames = connectorNames; } public PageParams getPageParams() { return pageParams; } + public List getIndexNames() { + return indexNames; + } + + public List getConnectorNames() { + return connectorNames; + } + @Override public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; // Pagination validation is done as part of PageParams constructor - return null; + + if (indexNames != null && indexNames.isEmpty() == false) { + for (String indexName : indexNames) { + try { + MetadataCreateIndexService.validateIndexOrAliasName(indexName, InvalidIndexNameException::new); + } catch (InvalidIndexNameException e) { + validationException = addValidationError(e.toString(), validationException); + } + } + } + return validationException; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); pageParams.writeTo(out); + out.writeOptionalStringCollection(indexNames); + out.writeOptionalStringCollection(connectorNames); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ListConnectorAction.Request that = (ListConnectorAction.Request) o; - return Objects.equals(pageParams, that.pageParams); + ListConnectorAction.Request request = (ListConnectorAction.Request) o; + return Objects.equals(pageParams, request.pageParams) + && Objects.equals(indexNames, request.indexNames) + && Objects.equals(connectorNames, request.connectorNames); } @Override public int hashCode() { - return Objects.hash(pageParams); + return Objects.hash(pageParams, indexNames, connectorNames); } + @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "list_connector_request", - p -> new ListConnectorAction.Request((PageParams) p[0]) + p -> new ListConnectorAction.Request((PageParams) p[0], (List) p[1], (List) p[2]) ); static { PARSER.declareObject(constructorArg(), (p, c) -> PageParams.fromXContent(p), PAGE_PARAMS_FIELD); + PARSER.declareStringArray(optionalConstructorArg(), INDEX_NAMES_FIELD); + PARSER.declareStringArray(optionalConstructorArg(), NAMES_FIELD); } public static ListConnectorAction.Request parse(XContentParser parser) { @@ -95,7 +133,11 @@ public static ListConnectorAction.Request parse(XContentParser parser) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(PAGE_PARAMS_FIELD.getPreferredName(), pageParams); + { + builder.field(PAGE_PARAMS_FIELD.getPreferredName(), pageParams); + builder.field(INDEX_NAMES_FIELD.getPreferredName(), indexNames); + builder.field(NAMES_FIELD.getPreferredName(), connectorNames); + } builder.endObject(); return builder; } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java index 9c37e31944ac8..90232b340719d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.application.EnterpriseSearch; +import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.core.action.util.PageParams; import java.io.IOException; @@ -38,7 +39,10 @@ public List routes() { protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { int from = restRequest.paramAsInt("from", PageParams.DEFAULT_FROM); int size = restRequest.paramAsInt("size", PageParams.DEFAULT_SIZE); - ListConnectorAction.Request request = new ListConnectorAction.Request(new PageParams(from, size)); + List indexNames = List.of(restRequest.paramAsStringArray(Connector.INDEX_NAME_FIELD.getPreferredName(), new String[0])); + List connectorNames = List.of(restRequest.paramAsStringArray("connector_name", new String[0])); + + ListConnectorAction.Request request = new ListConnectorAction.Request(new PageParams(from, size), indexNames, connectorNames); return channel -> client.execute(ListConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorApiKeyIdAction.java new file mode 100644 index 0000000000000..0cb42f6f448a2 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorApiKeyIdAction.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestUpdateConnectorApiKeyIdAction extends BaseRestHandler { + + @Override + public String getName() { + return "connector_update_api_key_id_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/" + EnterpriseSearch.CONNECTOR_API_ENDPOINT + "/{connector_id}/_api_key_id")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + UpdateConnectorApiKeyIdAction.Request request = UpdateConnectorApiKeyIdAction.Request.fromXContentBytes( + restRequest.param("connector_id"), + restRequest.content(), + restRequest.getXContentType() + ); + return channel -> client.execute( + UpdateConnectorApiKeyIdAction.INSTANCE, + request, + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorIndexNameAction.java new file mode 100644 index 0000000000000..ce6dd0a5ba24f --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorIndexNameAction.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestUpdateConnectorIndexNameAction extends BaseRestHandler { + + @Override + public String getName() { + return "connector_update_index_name_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/" + EnterpriseSearch.CONNECTOR_API_ENDPOINT + "/{connector_id}/_index_name")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + UpdateConnectorIndexNameAction.Request request = UpdateConnectorIndexNameAction.Request.fromXContentBytes( + restRequest.param("connector_id"), + restRequest.content(), + restRequest.getXContentType() + ); + return channel -> client.execute( + UpdateConnectorIndexNameAction.INSTANCE, + request, + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorStatusAction.java new file mode 100644 index 0000000000000..9770a051ce4fc --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorStatusAction.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestUpdateConnectorStatusAction extends BaseRestHandler { + + @Override + public String getName() { + return "connector_update_status_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/" + EnterpriseSearch.CONNECTOR_API_ENDPOINT + "/{connector_id}/_status")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + UpdateConnectorStatusAction.Request request = UpdateConnectorStatusAction.Request.fromXContentBytes( + restRequest.param("connector_id"), + restRequest.content(), + restRequest.getXContentType() + ); + return channel -> client.execute( + UpdateConnectorStatusAction.INSTANCE, + request, + new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java index cfe05965da37b..03334751c5a42 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java @@ -42,9 +42,12 @@ public TransportListConnectorAction( @Override protected void doExecute(Task task, ListConnectorAction.Request request, ActionListener listener) { final PageParams pageParams = request.getPageParams(); + connectorIndexService.listConnectors( pageParams.getFrom(), pageParams.getSize(), + request.getIndexNames(), + request.getConnectorNames(), listener.map(r -> new ListConnectorAction.Response(r.connectors(), r.totalResults())) ); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorApiKeyIdAction.java new file mode 100644 index 0000000000000..32cd8494fa391 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorApiKeyIdAction.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; + +public class TransportUpdateConnectorApiKeyIdAction extends HandledTransportAction< + UpdateConnectorApiKeyIdAction.Request, + ConnectorUpdateActionResponse> { + + protected final ConnectorIndexService connectorIndexService; + + @Inject + public TransportUpdateConnectorApiKeyIdAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client + ) { + super( + UpdateConnectorApiKeyIdAction.NAME, + transportService, + actionFilters, + UpdateConnectorApiKeyIdAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorIndexService = new ConnectorIndexService(client); + } + + @Override + protected void doExecute( + Task task, + UpdateConnectorApiKeyIdAction.Request request, + ActionListener listener + ) { + connectorIndexService.updateConnectorApiKeyIdOrApiKeySecretId( + request, + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorIndexNameAction.java new file mode 100644 index 0000000000000..b582e9b740877 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorIndexNameAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; + +public class TransportUpdateConnectorIndexNameAction extends HandledTransportAction< + UpdateConnectorIndexNameAction.Request, + ConnectorUpdateActionResponse> { + + protected final ConnectorIndexService connectorIndexService; + + @Inject + public TransportUpdateConnectorIndexNameAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client + ) { + super( + UpdateConnectorIndexNameAction.NAME, + transportService, + actionFilters, + UpdateConnectorIndexNameAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorIndexService = new ConnectorIndexService(client); + } + + @Override + protected void doExecute( + Task task, + UpdateConnectorIndexNameAction.Request request, + ActionListener listener + ) { + connectorIndexService.updateConnectorIndexName(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorStatusAction.java new file mode 100644 index 0000000000000..c9ea60b6149e0 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorStatusAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; + +public class TransportUpdateConnectorStatusAction extends HandledTransportAction< + UpdateConnectorStatusAction.Request, + ConnectorUpdateActionResponse> { + + protected final ConnectorIndexService connectorIndexService; + + @Inject + public TransportUpdateConnectorStatusAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client + ) { + super( + UpdateConnectorStatusAction.NAME, + transportService, + actionFilters, + UpdateConnectorStatusAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorIndexService = new ConnectorIndexService(client); + } + + @Override + protected void doExecute( + Task task, + UpdateConnectorStatusAction.Request request, + ActionListener listener + ) { + connectorIndexService.updateConnectorStatus(request, listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java new file mode 100644 index 0000000000000..f69d632950ae6 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.Connector; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class UpdateConnectorApiKeyIdAction { + + public static final String NAME = "cluster:admin/xpack/connector/update_api_key_id"; + public static final ActionType INSTANCE = new ActionType<>(NAME); + + private UpdateConnectorApiKeyIdAction() {/* no instances */} + + public static class Request extends ActionRequest implements ToXContentObject { + + private final String connectorId; + + @Nullable + private final String apiKeyId; + + @Nullable + private final String apiKeySecretId; + + public Request(String connectorId, String apiKeyId, String apiKeySecretId) { + this.connectorId = connectorId; + this.apiKeyId = apiKeyId; + this.apiKeySecretId = apiKeySecretId; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorId = in.readString(); + this.apiKeyId = in.readOptionalString(); + this.apiKeySecretId = in.readOptionalString(); + } + + public String getConnectorId() { + return connectorId; + } + + public String getApiKeyId() { + return apiKeyId; + } + + public String getApiKeySecretId() { + return apiKeySecretId; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(connectorId)) { + validationException = addValidationError("[connector_id] cannot be [null] or [\"\"].", validationException); + } + if (apiKeyId == null && apiKeySecretId == null) { + validationException = addValidationError( + "[api_key_id] and [api_key_secret_id] cannot both be [null]. Please provide a value for at least one of them.", + validationException + ); + } + + return validationException; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_update_api_key_id_request", + false, + ((args, connectorId) -> new UpdateConnectorApiKeyIdAction.Request(connectorId, (String) args[0], (String) args[1])) + ); + + static { + PARSER.declareStringOrNull(optionalConstructorArg(), Connector.API_KEY_ID_FIELD); + PARSER.declareStringOrNull(optionalConstructorArg(), Connector.API_KEY_SECRET_ID_FIELD); + } + + public static UpdateConnectorApiKeyIdAction.Request fromXContentBytes( + String connectorId, + BytesReference source, + XContentType xContentType + ) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return UpdateConnectorApiKeyIdAction.Request.fromXContent(parser, connectorId); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e); + } + } + + public static UpdateConnectorApiKeyIdAction.Request fromXContent(XContentParser parser, String connectorId) throws IOException { + return PARSER.parse(parser, connectorId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + if (apiKeyId != null) { + builder.field(Connector.API_KEY_ID_FIELD.getPreferredName(), apiKeyId); + } + if (apiKeySecretId != null) { + builder.field(Connector.API_KEY_SECRET_ID_FIELD.getPreferredName(), apiKeySecretId); + } + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorId); + out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeySecretId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorId, request.connectorId) + && Objects.equals(apiKeyId, request.apiKeyId) + && Objects.equals(apiKeySecretId, request.apiKeySecretId); + } + + @Override + public int hashCode() { + return Objects.hash(connectorId, apiKeyId, apiKeySecretId); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java new file mode 100644 index 0000000000000..e793ccac5669c --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.indices.InvalidIndexNameException; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.Connector; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + +public class UpdateConnectorIndexNameAction { + + public static final String NAME = "cluster:admin/xpack/connector/update_index_name"; + public static final ActionType INSTANCE = new ActionType<>(NAME); + + private UpdateConnectorIndexNameAction() {/* no instances */} + + public static class Request extends ActionRequest implements ToXContentObject { + + private final String connectorId; + private final String indexName; + + public Request(String connectorId, String indexName) { + this.connectorId = connectorId; + this.indexName = indexName; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorId = in.readString(); + this.indexName = in.readString(); + } + + public String getConnectorId() { + return connectorId; + } + + public String getIndexName() { + return indexName; + } + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "connector_update_index_name_request", + false, + ((args, connectorId) -> new UpdateConnectorIndexNameAction.Request(connectorId, (String) args[0])) + ); + + static { + PARSER.declareString(constructorArg(), Connector.INDEX_NAME_FIELD); + } + + public static UpdateConnectorIndexNameAction.Request fromXContentBytes( + String connectorId, + BytesReference source, + XContentType xContentType + ) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return UpdateConnectorIndexNameAction.Request.fromXContent(parser, connectorId); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e); + } + } + + public static UpdateConnectorIndexNameAction.Request fromXContent(XContentParser parser, String connectorId) throws IOException { + return PARSER.parse(parser, connectorId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(Connector.INDEX_NAME_FIELD.getPreferredName(), indexName); + } + builder.endObject(); + return builder; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(connectorId)) { + validationException = addValidationError("[connector_id] cannot be [null] or [\"\"].", validationException); + } + + if (Strings.isNullOrEmpty(indexName)) { + validationException = addValidationError("[index_name] cannot be [null] or [\"\"].", validationException); + } + + try { + MetadataCreateIndexService.validateIndexOrAliasName(indexName, InvalidIndexNameException::new); + } catch (InvalidIndexNameException e) { + validationException = addValidationError(e.toString(), validationException); + } + + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorId); + out.writeString(indexName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorId, request.connectorId) && Objects.equals(indexName, request.indexName); + } + + @Override + public int hashCode() { + return Objects.hash(connectorId, indexName); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java new file mode 100644 index 0000000000000..256dff37ab9b8 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorStatus; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class UpdateConnectorStatusAction { + + public static final String NAME = "cluster:admin/xpack/connector/update_status"; + public static final ActionType INSTANCE = new ActionType<>(NAME); + + public UpdateConnectorStatusAction() {/* no instances */} + + public static class Request extends ActionRequest implements ToXContentObject { + + private final String connectorId; + private final ConnectorStatus status; + + public Request(String connectorId, ConnectorStatus status) { + this.connectorId = connectorId; + this.status = status; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorId = in.readString(); + this.status = in.readEnum(ConnectorStatus.class); + } + + public String getConnectorId() { + return connectorId; + } + + public ConnectorStatus getStatus() { + return status; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_update_status_request", + false, + ((args, connectorId) -> new UpdateConnectorStatusAction.Request(connectorId, (ConnectorStatus) args[0])) + ); + + static { + PARSER.declareField( + optionalConstructorArg(), + (p, c) -> ConnectorStatus.connectorStatus(p.text()), + Connector.STATUS_FIELD, + ObjectParser.ValueType.STRING + ); + } + + public static UpdateConnectorStatusAction.Request fromXContentBytes( + String connectorId, + BytesReference source, + XContentType xContentType + ) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return UpdateConnectorStatusAction.Request.fromXContent(parser, connectorId); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e); + } + } + + public static UpdateConnectorStatusAction.Request fromXContent(XContentParser parser, String connectorId) throws IOException { + return PARSER.parse(parser, connectorId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(Connector.STATUS_FIELD.getPreferredName(), status); + } + builder.endObject(); + return builder; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(connectorId)) { + validationException = addValidationError("[connector_id] cannot be [null] or [\"\"].", validationException); + } + + if (status == null) { + validationException = addValidationError("[status] cannot be [null].", validationException); + } + + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorId); + out.writeEnum(status); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorId, request.connectorId) && status == request.status; + } + + @Override + public int hashCode() { + return Objects.hash(connectorId, status); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java index fb34035e5400b..c531187dbb0a0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java @@ -322,7 +322,7 @@ public ConnectorSyncJob(StreamInput in) throws IOException { STATUS_FIELD, ObjectParser.ValueType.STRING ); - PARSER.declareLong(constructorArg(), TOTAL_DOCUMENT_COUNT_FIELD); + PARSER.declareLongOrNull(constructorArg(), 0L, TOTAL_DOCUMENT_COUNT_FIELD); PARSER.declareField( constructorArg(), (p, c) -> ConnectorSyncJobTriggerMethod.fromString(p.text()), diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java index 67c918dac94c9..4535365f42000 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/LocalStateEnterpriseSearch.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.license.XPackLicenseState; @@ -28,6 +29,7 @@ import java.nio.file.Path; import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class LocalStateEnterpriseSearch extends LocalStateCompositeXPackPlugin { @@ -77,7 +79,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return entSearchPlugin.getRestHandlers( settings, @@ -87,7 +90,8 @@ public List getRestHandlers( indexScopedSettings, settingsFilter, indexNameExpressionResolver, - nodesInCluster + nodesInCluster, + clusterSupportsFeature ); } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorConfigurationTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorConfigurationTests.java index 9b1f9c60d1607..35b21ce676a57 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorConfigurationTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorConfigurationTests.java @@ -85,6 +85,46 @@ public void testToXContent() throws IOException { assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); } + public void testToXContentCrawlerConfig_WithNullValue() throws IOException { + String content = XContentHelper.stripWhitespace(""" + { + "label": "nextSyncConfig", + "value": null + } + """); + + ConnectorConfiguration configuration = ConnectorConfiguration.fromXContentBytes(new BytesArray(content), XContentType.JSON); + boolean humanReadable = true; + BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); + ConnectorConfiguration parsed; + try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) { + parsed = ConnectorConfiguration.fromXContent(parser); + } + assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); + } + + public void testToXContentCrawlerConfig_WithCrawlerConfigurationOverrides() throws IOException { + String content = XContentHelper.stripWhitespace(""" + { + "label": "nextSyncConfig", + "value": { + "max_crawl_depth": 3, + "sitemap_discovery_disabled": false, + "seed_urls": ["https://elastic.co/"] + } + } + """); + + ConnectorConfiguration configuration = ConnectorConfiguration.fromXContentBytes(new BytesArray(content), XContentType.JSON); + boolean humanReadable = true; + BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); + ConnectorConfiguration parsed; + try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) { + parsed = ConnectorConfiguration.fromXContent(parser); + } + assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); + } + public void testToXContentWithMultipleConstraintTypes() throws IOException { String content = XContentHelper.stripWhitespace(""" { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index c043bfd4453d8..52bfd64db1844 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.application.connector; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; @@ -24,9 +25,11 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.action.PostConnectorAction; import org.elasticsearch.xpack.application.connector.action.PutConnectorAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorApiKeyIdAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorConfigurationAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorErrorAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorIndexNameAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSeenAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorLastSyncStatsAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorNameAction; @@ -34,6 +37,7 @@ import org.elasticsearch.xpack.application.connector.action.UpdateConnectorPipelineAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorSchedulingAction; import org.elasticsearch.xpack.application.connector.action.UpdateConnectorServiceTypeAction; +import org.elasticsearch.xpack.application.connector.action.UpdateConnectorStatusAction; import org.junit.Before; import java.util.ArrayList; @@ -231,6 +235,43 @@ public void testUpdateConnectorScheduling() throws Exception { assertThat(updatedScheduling, equalTo(indexedConnector.getScheduling())); } + public void testUpdateConnectorIndexName() throws Exception { + Connector connector = ConnectorTestUtils.getRandomConnector(); + String connectorId = randomUUID(); + + DocWriteResponse resp = buildRequestAndAwaitPutConnector(connectorId, connector); + assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK))); + + String newIndexName = randomAlphaOfLengthBetween(3, 10); + + UpdateConnectorIndexNameAction.Request updateIndexNameRequest = new UpdateConnectorIndexNameAction.Request( + connectorId, + newIndexName + ); + + DocWriteResponse updateResponse = awaitUpdateConnectorIndexName(updateIndexNameRequest); + assertThat(updateResponse.status(), equalTo(RestStatus.OK)); + + Connector indexedConnector = awaitGetConnector(connectorId); + assertThat(newIndexName, equalTo(indexedConnector.getIndexName())); + } + + public void testUpdateConnectorIndexName_WithTheSameIndexName() throws Exception { + Connector connector = ConnectorTestUtils.getRandomConnector(); + String connectorId = randomUUID(); + + DocWriteResponse resp = buildRequestAndAwaitPutConnector(connectorId, connector); + assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK))); + + UpdateConnectorIndexNameAction.Request updateIndexNameRequest = new UpdateConnectorIndexNameAction.Request( + connectorId, + connector.getIndexName() + ); + + DocWriteResponse updateResponse = awaitUpdateConnectorIndexName(updateIndexNameRequest); + assertThat(updateResponse.getResult(), equalTo(DocWriteResponse.Result.NOOP)); + } + public void testUpdateConnectorServiceType() throws Exception { Connector connector = ConnectorTestUtils.getRandomConnector(); String connectorId = randomUUID(); @@ -308,6 +349,60 @@ public void testUpdateConnectorNative() throws Exception { assertThat(isNative, equalTo(indexedConnector.isNative())); } + public void testUpdateConnectorStatus() throws Exception { + Connector connector = ConnectorTestUtils.getRandomConnector(); + String connectorId = randomUUID(); + + DocWriteResponse resp = buildRequestAndAwaitPutConnector(connectorId, connector); + assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK))); + + Connector indexedConnector = awaitGetConnector(connectorId); + + ConnectorStatus newStatus = ConnectorTestUtils.getRandomConnectorNextStatus(indexedConnector.getStatus()); + + UpdateConnectorStatusAction.Request updateStatusRequest = new UpdateConnectorStatusAction.Request(connectorId, newStatus); + + DocWriteResponse updateResponse = awaitUpdateConnectorStatus(updateStatusRequest); + assertThat(updateResponse.status(), equalTo(RestStatus.OK)); + + indexedConnector = awaitGetConnector(connectorId); + assertThat(newStatus, equalTo(indexedConnector.getStatus())); + } + + public void testUpdateConnectorStatus_WithInvalidStatus() throws Exception { + Connector connector = ConnectorTestUtils.getRandomConnector(); + String connectorId = randomUUID(); + + DocWriteResponse resp = buildRequestAndAwaitPutConnector(connectorId, connector); + Connector indexedConnector = awaitGetConnector(connectorId); + + ConnectorStatus newInvalidStatus = ConnectorTestUtils.getRandomInvalidConnectorNextStatus(indexedConnector.getStatus()); + + UpdateConnectorStatusAction.Request updateStatusRequest = new UpdateConnectorStatusAction.Request(connectorId, newInvalidStatus); + + expectThrows(ElasticsearchStatusException.class, () -> awaitUpdateConnectorStatus(updateStatusRequest)); + } + + public void testUpdateConnectorApiKeyIdOrApiKeySecretId() throws Exception { + Connector connector = ConnectorTestUtils.getRandomConnector(); + String connectorId = randomUUID(); + DocWriteResponse resp = buildRequestAndAwaitPutConnector(connectorId, connector); + assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK))); + + UpdateConnectorApiKeyIdAction.Request updateApiKeyIdRequest = new UpdateConnectorApiKeyIdAction.Request( + connectorId, + randomAlphaOfLengthBetween(5, 15), + randomAlphaOfLengthBetween(5, 15) + ); + + DocWriteResponse updateResponse = awaitUpdateConnectorApiKeyIdOrApiKeySecretId(updateApiKeyIdRequest); + assertThat(updateResponse.status(), equalTo(RestStatus.OK)); + + Connector indexedConnector = awaitGetConnector(connectorId); + assertThat(updateApiKeyIdRequest.getApiKeyId(), equalTo(indexedConnector.getApiKeyId())); + assertThat(updateApiKeyIdRequest.getApiKeySecretId(), equalTo(indexedConnector.getApiKeySecretId())); + } + private DeleteResponse awaitDeleteConnector(String connectorId) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); @@ -439,11 +534,12 @@ public void onFailure(Exception e) { return resp.get(); } - private ConnectorIndexService.ConnectorResult awaitListConnector(int from, int size) throws Exception { + private ConnectorIndexService.ConnectorResult awaitListConnector(int from, int size, List indexNames, List names) + throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); - connectorIndexService.listConnectors(from, size, new ActionListener<>() { + connectorIndexService.listConnectors(from, size, indexNames, names, new ActionListener<>() { @Override public void onResponse(ConnectorIndexService.ConnectorResult result) { resp.set(result); @@ -517,6 +613,60 @@ public void onFailure(Exception e) { return resp.get(); } + private UpdateResponse awaitUpdateConnectorIndexName(UpdateConnectorIndexNameAction.Request updateIndexNameRequest) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + connectorIndexService.updateConnectorIndexName(updateIndexNameRequest, new ActionListener<>() { + @Override + public void onResponse(UpdateResponse indexResponse) { + resp.set(indexResponse); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + }); + + assertTrue("Timeout waiting for update index name request", latch.await(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from update index name request", resp.get()); + + return resp.get(); + } + + private UpdateResponse awaitUpdateConnectorStatus(UpdateConnectorStatusAction.Request updateStatusRequest) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + connectorIndexService.updateConnectorStatus(updateStatusRequest, new ActionListener<>() { + @Override + public void onResponse(UpdateResponse indexResponse) { + resp.set(indexResponse); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + }); + + assertTrue("Timeout waiting for update status request", latch.await(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from update status request", resp.get()); + + return resp.get(); + } + private UpdateResponse awaitUpdateConnectorLastSeen(UpdateConnectorLastSeenAction.Request checkIn) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); @@ -719,6 +869,33 @@ public void onFailure(Exception e) { return resp.get(); } + private UpdateResponse awaitUpdateConnectorApiKeyIdOrApiKeySecretId( + UpdateConnectorApiKeyIdAction.Request updatedApiKeyIdOrApiKeySecretId + ) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + connectorIndexService.updateConnectorApiKeyIdOrApiKeySecretId(updatedApiKeyIdOrApiKeySecretId, new ActionListener<>() { + @Override + public void onResponse(UpdateResponse indexResponse) { + resp.set(indexResponse); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + }); + assertTrue("Timeout waiting for update api key id request", latch.await(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from update api key id request", resp.get()); + return resp.get(); + } + /** * Update configuration action is handled via painless script. This implementation mocks the painless script engine * for unit tests. diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachineTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachineTests.java index 6fdda83244db8..372c874310162 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachineTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorStateMachineTests.java @@ -28,7 +28,6 @@ public void testValidTransitionFromNeedsConfiguration() { public void testInvalidTransitionFromNeedsConfiguration() { assertFalse(ConnectorStateMachine.isValidTransition(ConnectorStatus.NEEDS_CONFIGURATION, ConnectorStatus.CREATED)); assertFalse(ConnectorStateMachine.isValidTransition(ConnectorStatus.NEEDS_CONFIGURATION, ConnectorStatus.CONNECTED)); - assertFalse(ConnectorStateMachine.isValidTransition(ConnectorStatus.NEEDS_CONFIGURATION, ConnectorStatus.ERROR)); } public void testValidTransitionFromConfigured() { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java index af0e9c5c48424..583caf88eeba7 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java @@ -30,11 +30,13 @@ import java.io.IOException; import java.time.Instant; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; @@ -247,6 +249,7 @@ public static Map getRandomConnectorConfiguratio public static Connector getRandomConnector() { return new Connector.Builder().setApiKeyId(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setApiKeySecretId(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setConfiguration(getRandomConnectorConfiguration()) .setCustomScheduling(Map.of(randomAlphaOfLengthBetween(5, 10), getRandomConnectorCustomSchedule())) .setDescription(randomFrom(new String[] { null, randomAlphaOfLength(10) })) @@ -261,7 +264,7 @@ public static Connector getRandomConnector() { .setName(randomFrom(new String[] { null, randomAlphaOfLength(10) })) .setPipeline(randomBoolean() ? getRandomConnectorIngestPipeline() : null) .setScheduling(getRandomConnectorScheduling()) - .setStatus(getRandomConnectorStatus()) + .setStatus(getRandomConnectorInitialStatus()) .setSyncCursor(randomBoolean() ? Map.of(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)) : null) .setSyncNow(randomBoolean()) .build(); @@ -343,7 +346,23 @@ public static ConnectorSyncJobType getRandomSyncJobType() { return values[randomInt(values.length - 1)]; } - private static ConnectorStatus getRandomConnectorStatus() { + public static ConnectorStatus getRandomConnectorInitialStatus() { + return randomFrom(ConnectorStatus.CREATED, ConnectorStatus.NEEDS_CONFIGURATION); + } + + public static ConnectorStatus getRandomConnectorNextStatus(ConnectorStatus connectorStatus) { + return randomFrom(ConnectorStateMachine.validNextStates(connectorStatus)); + } + + public static ConnectorStatus getRandomInvalidConnectorNextStatus(ConnectorStatus connectorStatus) { + Set validNextStatus = ConnectorStateMachine.validNextStates(connectorStatus); + List invalidStatuses = Arrays.stream(ConnectorStatus.values()) + .filter(status -> validNextStatus.contains(status) == false) + .toList(); + return randomFrom(invalidStatuses); + } + + public static ConnectorStatus getRandomConnectorStatus() { ConnectorStatus[] values = ConnectorStatus.values(); return values[randomInt(values.length - 1)]; } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java index cdfa3dea8a6fa..0fd590a4ce106 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java @@ -50,7 +50,8 @@ public void testToXContent() throws IOException { String connectorId = "test-connector"; String content = XContentHelper.stripWhitespace(""" { - "api_key_id":"test", + "api_key_id":"test-aki", + "api_key_secret_id":"test-aksi", "custom_scheduling":{ "schedule-key":{ "configuration_overrides":{ @@ -246,6 +247,7 @@ public void testToContent_WithNullValues() throws IOException { String content = XContentHelper.stripWhitespace(""" { "api_key_id": null, + "api_key_secret_id": null, "custom_scheduling":{}, "configuration":{}, "description": null, @@ -299,6 +301,71 @@ public void testToContent_WithNullValues() throws IOException { assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); } + public void testToXContent_withOptionalFieldsMissing() throws IOException { + // This test is to ensure the doc can serialize without fields that have been added since 8.12. + // This is to avoid breaking serverless, which has a regular BC built + // that can be broken if we haven't made migrations yet. + String connectorId = "test-connector"; + + // Missing from doc: + // api_key_secret_id + String content = XContentHelper.stripWhitespace(""" + { + "api_key_id": null, + "custom_scheduling":{}, + "configuration":{}, + "description": null, + "features": null, + "filtering":[], + "index_name": "search-test", + "is_native": false, + "language": null, + "last_access_control_sync_error": null, + "last_access_control_sync_scheduled_at": null, + "last_access_control_sync_status": null, + "last_incremental_sync_scheduled_at": null, + "last_seen": null, + "last_sync_error": null, + "last_sync_scheduled_at": null, + "last_sync_status": null, + "last_synced": null, + "name": null, + "pipeline":{ + "extract_binary_content":true, + "name":"ent-search-generic-ingestion", + "reduce_whitespace":true, + "run_ml_inference":false + }, + "scheduling":{ + "access_control":{ + "enabled":false, + "interval":"0 0 0 * * ?" + }, + "full":{ + "enabled":false, + "interval":"0 0 0 * * ?" + }, + "incremental":{ + "enabled":false, + "interval":"0 0 0 * * ?" + } + }, + "service_type": null, + "status": "needs_configuration", + "sync_now":false + }"""); + + Connector connector = Connector.fromXContentBytes(new BytesArray(content), connectorId, XContentType.JSON); + boolean humanReadable = true; + BytesReference originalBytes = toShuffledXContent(connector, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); + Connector parsed; + try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) { + parsed = Connector.fromXContent(parser, connectorId); + } + assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); + assertThat(parsed.getApiKeySecretId(), equalTo(null)); + } + private void assertTransportSerialization(Connector testInstance) throws IOException { Connector deserializedInstance = copyInstance(testInstance); assertNotSame(testInstance, deserializedInstance); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java index b31c3e90b7403..3d2192098d907 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; +import java.util.List; public class ListConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { @Override @@ -25,7 +26,11 @@ protected Writeable.Reader instanceReader() { @Override protected ListConnectorAction.Request createTestInstance() { PageParams pageParams = SearchApplicationTestUtils.randomPageParams(); - return new ListConnectorAction.Request(pageParams); + return new ListConnectorAction.Request( + pageParams, + List.of(generateRandomStringArray(10, 10, false)), + List.of(generateRandomStringArray(10, 10, false)) + ); } @Override @@ -40,6 +45,6 @@ protected ListConnectorAction.Request doParseInstance(XContentParser parser) thr @Override protected ListConnectorAction.Request mutateInstanceForVersion(ListConnectorAction.Request instance, TransportVersion version) { - return new ListConnectorAction.Request(instance.getPageParams()); + return new ListConnectorAction.Request(instance.getPageParams(), instance.getIndexNames(), instance.getConnectorNames()); } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..a6671aedd9910 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdActionRequestBWCSerializingTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class UpdateConnectorApiKeyIdActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase< + UpdateConnectorApiKeyIdAction.Request> { + + private String connectorId; + + @Override + protected Writeable.Reader instanceReader() { + return UpdateConnectorApiKeyIdAction.Request::new; + } + + @Override + protected UpdateConnectorApiKeyIdAction.Request createTestInstance() { + this.connectorId = randomUUID(); + return new UpdateConnectorApiKeyIdAction.Request(connectorId, randomAlphaOfLengthBetween(5, 15), randomAlphaOfLengthBetween(5, 15)); + } + + @Override + protected UpdateConnectorApiKeyIdAction.Request mutateInstance(UpdateConnectorApiKeyIdAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected UpdateConnectorApiKeyIdAction.Request doParseInstance(XContentParser parser) throws IOException { + return UpdateConnectorApiKeyIdAction.Request.fromXContent(parser, this.connectorId); + } + + @Override + protected UpdateConnectorApiKeyIdAction.Request mutateInstanceForVersion( + UpdateConnectorApiKeyIdAction.Request instance, + TransportVersion version + ) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..99bf15d20385f --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameActionRequestBWCSerializingTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class UpdateConnectorIndexNameActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase< + UpdateConnectorIndexNameAction.Request> { + + private String connectorId; + + @Override + protected Writeable.Reader instanceReader() { + return UpdateConnectorIndexNameAction.Request::new; + } + + @Override + protected UpdateConnectorIndexNameAction.Request createTestInstance() { + this.connectorId = randomUUID(); + return new UpdateConnectorIndexNameAction.Request(connectorId, randomAlphaOfLengthBetween(3, 10)); + } + + @Override + protected UpdateConnectorIndexNameAction.Request mutateInstance(UpdateConnectorIndexNameAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected UpdateConnectorIndexNameAction.Request doParseInstance(XContentParser parser) throws IOException { + return UpdateConnectorIndexNameAction.Request.fromXContent(parser, this.connectorId); + } + + @Override + protected UpdateConnectorIndexNameAction.Request mutateInstanceForVersion( + UpdateConnectorIndexNameAction.Request instance, + TransportVersion version + ) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..b0efe0d8483ea --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusActionRequestBWCSerializingTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class UpdateConnectorStatusActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase< + UpdateConnectorStatusAction.Request> { + + private String connectorId; + + @Override + protected Writeable.Reader instanceReader() { + return UpdateConnectorStatusAction.Request::new; + } + + @Override + protected UpdateConnectorStatusAction.Request createTestInstance() { + this.connectorId = randomUUID(); + return new UpdateConnectorStatusAction.Request(connectorId, ConnectorTestUtils.getRandomConnectorStatus()); + } + + @Override + protected UpdateConnectorStatusAction.Request mutateInstance(UpdateConnectorStatusAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected UpdateConnectorStatusAction.Request doParseInstance(XContentParser parser) throws IOException { + return UpdateConnectorStatusAction.Request.fromXContent(parser, this.connectorId); + } + + @Override + protected UpdateConnectorStatusAction.Request mutateInstanceForVersion( + UpdateConnectorStatusAction.Request instance, + TransportVersion version + ) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java index 7b1a0f7d8dcf7..81b05ce25e177 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java @@ -235,7 +235,7 @@ public void testFromXContent_WithAllNullableFieldsSetToNull_DoesNotThrow() throw "metadata": {}, "started_at": null, "status": "canceling", - "total_document_count": 0, + "total_document_count": null, "trigger_method": "scheduled", "worker_hostname": null } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index f9f9238b6c4ab..5eef57cbb6c5b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -22,6 +22,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.InstantiatingObjectParser; @@ -43,6 +44,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex; public class EqlSearchResponse extends ActionResponse implements ToXContentObject, QlStatusResponse.AsyncStatus { @@ -260,8 +262,8 @@ private static final class Fields { private final boolean missing; - public Event(String index, String id, BytesReference source, Map fetchFields) { - this(index, id, source, fetchFields, false); + public Event(SearchHit hit) { + this(qualifiedIndex(hit), hit.getId(), hit.getSourceRef(), hit.getDocumentFields(), false); } public Event(String index, String id, BytesReference source, Map fetchFields, Boolean missing) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java index 0749b53c7b1cf..a7845ca62dccc 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java @@ -9,14 +9,12 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event; -import org.elasticsearch.xpack.eql.execution.search.RuntimeUtils; import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex; - public class EventPayload extends AbstractPayload { private final List values; @@ -24,10 +22,11 @@ public class EventPayload extends AbstractPayload { public EventPayload(SearchResponse response) { super(response.isTimedOut(), response.getTook()); - List hits = RuntimeUtils.searchHits(response); - values = new ArrayList<>(hits.size()); + SearchHits hits = response.getHits(); + values = new ArrayList<>(hits.getHits().length); for (SearchHit hit : hits) { - values.add(new Event(qualifiedIndex(hit), hit.getId(), hit.getSourceRef(), hit.getDocumentFields())); + // TODO: remove unpooled usage + values.add(new Event(hit.asUnpooled())); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/ReversePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/ReversePayload.java deleted file mode 100644 index 533dc3a992e74..0000000000000 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/ReversePayload.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.eql.execution.payload; - -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.xpack.eql.session.Payload; - -import java.util.Collections; -import java.util.List; - -public class ReversePayload implements Payload { - - private final Payload delegate; - - public ReversePayload(Payload delegate) { - this.delegate = delegate; - Collections.reverse(delegate.values()); - } - - @Override - public Type resultType() { - return delegate.resultType(); - } - - @Override - public boolean timedOut() { - return delegate.timedOut(); - } - - @Override - public TimeValue timeTook() { - return delegate.timeTook(); - } - - @Override - public List values() { - return delegate.values(); - } -} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java index ddd33e58f5448..121f4c208273b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java @@ -15,8 +15,6 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex; - class SamplePayload extends AbstractPayload { private final List values; @@ -30,7 +28,7 @@ class SamplePayload extends AbstractPayload { List hits = docs.get(i); List events = new ArrayList<>(hits.size()); for (SearchHit hit : hits) { - events.add(new Event(qualifiedIndex(hit), hit.getId(), hit.getSourceRef(), hit.getDocumentFields())); + events.add(new Event(hit)); } values.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(s.key().asList(), events)); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/AsEventListener.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/AsEventListener.java deleted file mode 100644 index 122e28f4e50b8..0000000000000 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/AsEventListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.eql.execution.search; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.DelegatingActionListener; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.xpack.eql.execution.payload.EventPayload; -import org.elasticsearch.xpack.eql.session.Payload; - -public class AsEventListener extends DelegatingActionListener { - - public AsEventListener(ActionListener listener) { - super(listener); - } - - @Override - public void onResponse(SearchResponse response) { - delegate.onResponse(new EventPayload(response)); - } -} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java index ceaf8bcbb6b6f..6cbe5298b5950 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -159,13 +159,13 @@ public void fetchHits(Iterable> refs, ActionListener docs = RuntimeUtils.searchHits(item.getResponse()); // for each doc, find its reference and its position inside the matrix - for (SearchHit doc : docs) { + for (SearchHit doc : item.getResponse().getHits()) { HitReference docRef = new HitReference(doc); List positions = referenceToPosition.get(docRef); positions.forEach(pos -> { - SearchHit previous = seq.get(pos / listSize).set(pos % listSize, doc); + // TODO: stop using unpooled + SearchHit previous = seq.get(pos / listSize).set(pos % listSize, doc.asUnpooled()); if (previous != null) { throw new EqlIllegalArgumentException( "Overriding sequence match [{}] with [{}]", diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/ReverseListener.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/ReverseListener.java deleted file mode 100644 index bc6fd8c82b85a..0000000000000 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/ReverseListener.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.eql.execution.search; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.DelegatingActionListener; -import org.elasticsearch.xpack.eql.execution.payload.ReversePayload; -import org.elasticsearch.xpack.eql.session.Payload; - -public class ReverseListener extends DelegatingActionListener { - - public ReverseListener(ActionListener delegate) { - super(delegate); - } - - @Override - public void onResponse(Payload response) { - delegate.onResponse(new ReversePayload(response)); - } -} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java index 95e18d54f5a08..45083babddbb4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java @@ -15,8 +15,6 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex; - class SequencePayload extends AbstractPayload { private final List values; @@ -33,7 +31,7 @@ class SequencePayload extends AbstractPayload { if (hit == null) { events.add(Event.MISSING_EVENT); } else { - events.add(new Event(qualifiedIndex(hit), hit.getId(), hit.getSourceRef(), hit.getDocumentFields())); + events.add(new Event(hit)); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java index d692bc376de01..35f171806ccb2 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java @@ -732,8 +732,7 @@ private void doPayload(ActionListener listener) { if (criteria.get(matcher.firstPositiveStage).descending()) { Collections.reverse(completed); } - SequencePayload payload = new SequencePayload(completed, addMissingEventPlaceholders(listOfHits), false, timeTook()); - return payload; + return new SequencePayload(completed, addMissingEventPlaceholders(listOfHits), false, timeTook()); })); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java index 4877b4d909a72..6fa61dcd84e48 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java @@ -11,10 +11,9 @@ import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.xpack.eql.execution.search.AsEventListener; +import org.elasticsearch.xpack.eql.execution.payload.EventPayload; import org.elasticsearch.xpack.eql.execution.search.BasicQueryClient; import org.elasticsearch.xpack.eql.execution.search.QueryRequest; -import org.elasticsearch.xpack.eql.execution.search.ReverseListener; import org.elasticsearch.xpack.eql.execution.search.SourceGenerator; import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.eql.session.EqlConfiguration; @@ -24,6 +23,7 @@ import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -71,8 +71,11 @@ public SearchSourceBuilder source(EqlSession session, boolean includeFetchFields public void execute(EqlSession session, ActionListener listener) { // endpoint - fetch all source QueryRequest request = () -> source(session, true).fetchSource(FetchSourceContext.FETCH_SOURCE); - listener = shouldReverse(request) ? new ReverseListener(listener) : listener; - new BasicQueryClient(session).query(request, new AsEventListener(listener)); + new BasicQueryClient(session).query(request, listener.safeMap(shouldReverse(request) ? r -> { + var res = new EventPayload(r); + Collections.reverse(res.values()); + return res; + } : EventPayload::new)); } private static boolean shouldReverse(QueryRequest query) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index fe21051b4063e..084a5e74a47e8 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.breaker.BreakerSettings; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.monitor.jvm.JvmInfo; @@ -42,6 +43,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class EqlPlugin extends Plugin implements ActionPlugin, CircuitBreakerPlugin { @@ -108,7 +110,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of( diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java index 255e94d6bda34..6cb283d11848e 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java @@ -117,7 +117,7 @@ static List randomEvents(XContentType xType) { if (fetchFields.isEmpty() && randomBoolean()) { fetchFields = null; } - hits.add(new Event(String.valueOf(i), randomAlphaOfLength(10), bytes, fetchFields)); + hits.add(new Event(String.valueOf(i), randomAlphaOfLength(10), bytes, fetchFields, false)); } } } @@ -297,7 +297,7 @@ private List mutateEvents(List original, TransportVersion version) } public void testEmptyIndexAsMissingEvent() throws IOException { - Event event = new Event("", "", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), null); + Event event = new Event("", "", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), null, false); BytesStreamOutput out = new BytesStreamOutput(); out.setTransportVersion(TransportVersions.V_8_9_X);// 8.9.1 event.writeTo(out); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorStatusTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorStatusTests.java index 01c5273a1e617..6c787052a8ae7 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorStatusTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorStatusTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.compute.lucene; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -19,7 +18,6 @@ import static org.hamcrest.Matchers.equalTo; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103774") public class LuceneSourceOperatorStatusTests extends AbstractWireSerializingTestCase { public static LuceneSourceOperator.Status simple() { return new LuceneSourceOperator.Status(2, Set.of("*:*"), new TreeSet<>(List.of("a:0", "a:1")), 0, 1, 5, 123, 99990, 8000); @@ -101,7 +99,7 @@ protected LuceneSourceOperator.Status mutateInstance(LuceneSourceOperator.Status switch (between(0, 8)) { case 0 -> processedSlices = randomValueOtherThan(processedSlices, ESTestCase::randomNonNegativeInt); case 1 -> processedQueries = randomValueOtherThan(processedQueries, LuceneSourceOperatorStatusTests::randomProcessedQueries); - case 2 -> processedQueries = randomValueOtherThan(processedShards, LuceneSourceOperatorStatusTests::randomProcessedShards); + case 2 -> processedShards = randomValueOtherThan(processedShards, LuceneSourceOperatorStatusTests::randomProcessedShards); case 3 -> sliceIndex = randomValueOtherThan(sliceIndex, ESTestCase::randomNonNegativeInt); case 4 -> totalSlices = randomValueOtherThan(totalSlices, ESTestCase::randomNonNegativeInt); case 5 -> pagesEmitted = randomValueOtherThan(pagesEmitted, ESTestCase::randomNonNegativeInt); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 408f58fb191b5..36e291905cf4e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.analysis.Verifier; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; @@ -41,6 +42,7 @@ import java.util.Set; import static java.util.Collections.emptyList; +import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.xpack.ql.TestUtils.of; import static org.hamcrest.Matchers.instanceOf; @@ -187,4 +189,48 @@ public static List withDefaultLimitWarning(List warnings) { return result; } + /** + * Generates a random enrich command with or without explicit parameters + */ + public static String randomEnrichCommand(String name, Enrich.Mode mode, String matchField, List enrichFields) { + String onField = " "; + String withFields = " "; + + List before = new ArrayList<>(); + List after = new ArrayList<>(); + + if (randomBoolean()) { + // => RENAME new_match_field=match_field | ENRICH name ON new_match_field | RENAME new_match_field AS match_field + String newMatchField = "my_" + matchField; + before.add("RENAME " + matchField + " AS " + newMatchField); + onField = " ON " + newMatchField; + after.add("RENAME " + newMatchField + " AS " + matchField); + } else if (randomBoolean()) { + onField = " ON " + matchField; + } + if (randomBoolean()) { + List fields = new ArrayList<>(); + for (String f : enrichFields) { + if (randomBoolean()) { + fields.add(f); + } else { + // ENRICH name WITH new_a=a,b|new_c=c | RENAME new_a AS a | RENAME new_c AS c + fields.add("new_" + f + "=" + f); + after.add("RENAME new_" + f + " AS " + f); + } + } + withFields = " WITH " + String.join(",", fields); + } + String enrich = "ENRICH"; + if (mode != Enrich.Mode.ANY || randomBoolean()) { + enrich += " [ccq.mode: " + mode + "] "; + } + enrich += " " + name; + enrich += onField; + enrich += withFields; + List all = new ArrayList<>(before); + all.add(enrich); + all.addAll(after); + return String.join(" | ", all); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java index 45753032997ce..59b684cdaa2cf 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java @@ -34,6 +34,8 @@ import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction; import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction; import org.elasticsearch.xpack.enrich.EnrichPlugin; +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.analysis.VerificationException; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.junit.After; @@ -81,6 +83,9 @@ protected Settings nodeSettings() { return Settings.builder().put(super.nodeSettings()).put(XPackSettings.SECURITY_ENABLED.getKey(), false).build(); } + static final EnrichPolicy hostPolicy = new EnrichPolicy("match", null, List.of("hosts"), "ip", List.of("ip", "os")); + static final EnrichPolicy vendorPolicy = new EnrichPolicy("match", null, List.of("vendors"), "os", List.of("os", "vendor")); + @Before public void setupHostsEnrich() { // the hosts policy are identical on every node @@ -113,8 +118,7 @@ public void setupHostsEnrich() { client.prepareIndex("hosts").setSource("ip", h.getKey(), "os", h.getValue()).get(); } client.admin().indices().prepareRefresh("hosts").get(); - EnrichPolicy policy = new EnrichPolicy("match", null, List.of("hosts"), "ip", List.of("ip", "os")); - client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("hosts", policy)).actionGet(); + client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("hosts", hostPolicy)).actionGet(); client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request("hosts")).actionGet(); assertAcked(client.admin().indices().prepareDelete("hosts")); } @@ -133,8 +137,7 @@ public void setupVendorPolicy() { client.prepareIndex("vendors").setSource("os", v.getKey(), "vendor", v.getValue()).get(); } client.admin().indices().prepareRefresh("vendors").get(); - EnrichPolicy policy = new EnrichPolicy("match", null, List.of("vendors"), "os", List.of("os", "vendor")); - client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("vendors", policy)).actionGet(); + client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("vendors", vendorPolicy)).actionGet(); client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request("vendors")).actionGet(); assertAcked(client.admin().indices().prepareDelete("vendors")); } @@ -197,17 +200,17 @@ public void wipeEnrichPolicies() { } } - static String enrichCommand(String policy, Enrich.Mode mode) { - if (mode == Enrich.Mode.ANY && randomBoolean()) { - return "ENRICH " + policy; - } - return "ENRICH[ccq.mode: " + mode + "] " + policy + " "; + static String enrichHosts(Enrich.Mode mode) { + return EsqlTestUtils.randomEnrichCommand("hosts", mode, hostPolicy.getMatchField(), hostPolicy.getEnrichFields()); + } + + static String enrichVendors(Enrich.Mode mode) { + return EsqlTestUtils.randomEnrichCommand("vendors", mode, vendorPolicy.getMatchField(), vendorPolicy.getEnrichFields()); } public void testWithHostsPolicy() { - for (var mode : List.of(Enrich.Mode.ANY, Enrich.Mode.COORDINATOR)) { - String enrich = enrichCommand("hosts", mode); - String query = "FROM events | eval ip= TO_STR(host) | " + enrich + " | stats c = COUNT(*) by os | SORT os"; + for (var mode : Enrich.Mode.values()) { + String query = "FROM events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; try (EsqlQueryResponse resp = runQuery(query)) { List> rows = getValuesList(resp); assertThat( @@ -224,9 +227,8 @@ public void testWithHostsPolicy() { ); } } - for (var mode : List.of(Enrich.Mode.ANY, Enrich.Mode.COORDINATOR)) { - String enrich = enrichCommand("hosts", mode); - String query = "FROM *:events | eval ip= TO_STR(host) | " + enrich + " | stats c = COUNT(*) by os | SORT os"; + for (var mode : Enrich.Mode.values()) { + String query = "FROM *:events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; try (EsqlQueryResponse resp = runQuery(query)) { List> rows = getValuesList(resp); assertThat( @@ -245,9 +247,8 @@ public void testWithHostsPolicy() { } } - for (var mode : List.of(Enrich.Mode.ANY, Enrich.Mode.COORDINATOR)) { - String enrich = enrichCommand("hosts", mode); - String query = "FROM *:events,events | eval ip= TO_STR(host) | " + enrich + " | stats c = COUNT(*) by os | SORT os"; + for (var mode : Enrich.Mode.values()) { + String query = "FROM *:events,events | eval ip= TO_STR(host) | " + enrichHosts(mode) + " | stats c = COUNT(*) by os | SORT os"; try (EsqlQueryResponse resp = runQuery(query)) { List> rows = getValuesList(resp); assertThat( @@ -267,10 +268,8 @@ public void testWithHostsPolicy() { } } - public void testEnrichHostsAggThenEnrichUsers() { - for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.COORDINATOR)) { - String enrichHosts = enrichCommand("hosts", hostMode); - String enrichVendors = enrichCommand("vendors", Enrich.Mode.COORDINATOR); + public void testEnrichHostsAggThenEnrichVendorCoordinator() { + for (var hostMode : Enrich.Mode.values()) { String query = String.format(Locale.ROOT, """ FROM *:events,events | eval ip= TO_STR(host) @@ -279,7 +278,7 @@ public void testEnrichHostsAggThenEnrichUsers() { | %s | stats c = SUM(c) by vendor | sort vendor - """, enrichHosts, enrichVendors); + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); try (EsqlQueryResponse resp = runQuery(query)) { assertThat( getValuesList(resp), @@ -298,15 +297,15 @@ public void testEnrichHostsAggThenEnrichUsers() { } public void testEnrichTwiceThenAggs() { - for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.COORDINATOR)) { + for (var hostMode : Enrich.Mode.values()) { String query = String.format(Locale.ROOT, """ FROM *:events,events | eval ip= TO_STR(host) - | ENRICH[ccq.mode:%s] hosts - | ENRICH[ccq.mode:coordinator] vendors + | %s + | %s | stats c = COUNT(*) by vendor | sort vendor - """, hostMode); + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); try (EsqlQueryResponse resp = runQuery(query)) { assertThat( getValuesList(resp), @@ -325,14 +324,14 @@ public void testEnrichTwiceThenAggs() { } public void testEnrichCoordinatorThenAny() { - String query = """ + String query = String.format(Locale.ROOT, """ FROM *:events,events | eval ip= TO_STR(host) - | ENRICH[ccq.mode:coordinator] hosts - | ENRICH vendors + | %s + | %s | stats c = COUNT(*) by vendor | sort vendor - """; + """, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.ANY)); try (EsqlQueryResponse resp = runQuery(query)) { assertThat( getValuesList(resp), @@ -349,15 +348,114 @@ public void testEnrichCoordinatorThenAny() { } } - public void testUnsupportedEnrichMode() { - for (Enrich.Mode mode : List.of(Enrich.Mode.REMOTE)) { - String enrich = enrichCommand("hosts", mode); - String q = "FROM *:events | eval ip= TO_STR(host) | " + enrich + " | stats c = COUNT(*) by os | SORT os"; - Exception error = expectThrows(IllegalArgumentException.class, () -> runQuery(q).close()); - assertThat(error.getMessage(), containsString("Enrich REMOTE mode is not supported yet")); + public void testEnrichCoordinatorWithVendor() { + for (Enrich.Mode hostMode : Enrich.Mode.values()) { + String query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.COORDINATOR)); + try (EsqlQueryResponse resp = runQuery(query)) { + assertThat( + getValuesList(resp), + equalTo( + List.of( + List.of(6L, "Apple"), + List.of(7L, "Microsoft"), + List.of(3L, "Redhat"), + List.of(3L, "Samsung"), + Arrays.asList(3L, (String) null) + ) + ) + ); + } + } + + } + + public void testEnrichRemoteWithVendor() { + for (Enrich.Mode hostMode : List.of(Enrich.Mode.ANY, Enrich.Mode.REMOTE)) { + var query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | %s + | stats c = COUNT(*) by vendor + | sort vendor + """, enrichHosts(hostMode), enrichVendors(Enrich.Mode.REMOTE)); + try (EsqlQueryResponse resp = runQuery(query)) { + assertThat( + getValuesList(resp), + equalTo( + List.of( + List.of(6L, "Apple"), + List.of(7L, "Microsoft"), + List.of(1L, "Redhat"), + List.of(2L, "Samsung"), + List.of(1L, "Sony"), + List.of(2L, "Suse"), + Arrays.asList(3L, (String) null) + ) + ) + ); + } } } + public void testTopNThenEnrichRemote() { + String query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | SORT ip + | LIMIT 5 + | %s + """, enrichHosts(Enrich.Mode.REMOTE)); + var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + assertThat(error.getMessage(), containsString("enrich with [ccq.mode:remote] can't be executed after LIMIT")); + } + + public void testLimitThenEnrichRemote() { + String query = String.format(Locale.ROOT, """ + FROM *:events,events + | LIMIT 10 + | eval ip= TO_STR(host) + | %s + """, enrichHosts(Enrich.Mode.REMOTE)); + var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + assertThat(error.getMessage(), containsString("enrich with [ccq.mode:remote] can't be executed after LIMIT")); + } + + public void testAggThenEnrichRemote() { + String query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | stats c = COUNT(*) by os + | %s + | sort vendor + """, enrichHosts(Enrich.Mode.ANY), enrichVendors(Enrich.Mode.REMOTE)); + var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + assertThat(error.getMessage(), containsString("enrich with [ccq.mode:remote] can't be executed after STATS")); + } + + public void testEnrichCoordinatorThenEnrichRemote() { + String query = String.format(Locale.ROOT, """ + FROM *:events,events + | eval ip= TO_STR(host) + | %s + | %s + | sort vendor + """, enrichHosts(Enrich.Mode.COORDINATOR), enrichVendors(Enrich.Mode.REMOTE)); + var error = expectThrows(VerificationException.class, () -> runQuery(query).close()); + assertThat( + error.getMessage(), + containsString("enrich with [ccq.mode:remote] can't be executed after another enrich with [ccq.mode:coordinator]") + ); + } + protected EsqlQueryResponse runQuery(String query) { EsqlQueryRequest request = new EsqlQueryRequest(); request.query(query); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index 1d2bff3cf360c..a589e1cc468a5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -39,6 +39,8 @@ import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction; import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction; import org.elasticsearch.xpack.enrich.EnrichPlugin; +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.junit.After; import org.junit.Before; @@ -50,6 +52,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -132,6 +135,8 @@ protected EsqlQueryResponse run(EsqlQueryRequest request) { return client.execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); } + static EnrichPolicy policy = new EnrichPolicy("match", null, List.of("songs"), "song_id", List.of("title", "artist", "length")); + @Before public void setupEnrichPolicies() { client().admin() @@ -152,7 +157,6 @@ record Song(String id, String title, String artist, double length) { client().prepareIndex("songs").setSource("song_id", s.id, "title", s.title, "artist", s.artist, "length", s.length).get(); } client().admin().indices().prepareRefresh("songs").get(); - EnrichPolicy policy = new EnrichPolicy("match", null, List.of("songs"), "song_id", List.of("title", "artist", "length")); client().execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("songs", policy)).actionGet(); client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request("songs")).actionGet(); assertAcked(client().admin().indices().prepareDelete("songs")); @@ -201,14 +205,12 @@ record Listen(long timestamp, String songId, double duration) { } private static String enrichSongCommand() { - String command = " ENRICH songs "; - if (randomBoolean()) { - command += " ON song_id "; - } - if (randomBoolean()) { - command += " WITH artist, title, length "; - } - return command; + return EsqlTestUtils.randomEnrichCommand( + "songs", + randomFrom(Enrich.Mode.COORDINATOR, Enrich.Mode.ANY), + policy.getMatchField(), + policy.getEnrichFields() + ); } public void testSumDurationByArtist() { @@ -316,6 +318,43 @@ public void testTopN() { } } + /** + * Some enrich queries that could fail without the PushDownEnrich rule. + */ + public void testForPushDownEnrichRule() { + { + String query = String.format(Locale.ROOT, """ + FROM listens* + | eval x = TO_STR(song_id) + | SORT x + | %s + | SORT song_id + | LIMIT 5 + | STATS listens = count(*) BY title + | SORT listens DESC + | KEEP title, listens + """, enrichSongCommand()); + try (EsqlQueryResponse resp = run(query)) { + assertThat(EsqlTestUtils.getValuesList(resp), equalTo(List.of(List.of("Hotel California", 3L), List.of("In The End", 2L)))); + } + } + { + String query = String.format(Locale.ROOT, """ + FROM listens* + | eval x = TO_STR(song_id) + | SORT x + | KEEP x, song_id + | %s + | SORT song_id + | KEEP title, song_id + | LIMIT 1 + """, enrichSongCommand()); + try (EsqlQueryResponse resp = run(query)) { + assertThat(EsqlTestUtils.getValuesList(resp), equalTo(List.of(List.of("Hotel California", "s1")))); + } + } + } + public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin { public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java index 070c0e112e051..ddce16857f6f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RestEsqlQueryAction.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.BaseRestHandler; @@ -36,11 +35,7 @@ public String getName() { @Override public List routes() { - return List.of( - new Route(POST, "/_query"), - // TODO: remove before release - Route.builder(POST, "/_esql").deprecated("_esql endpoint has been deprecated in favour of _query", RestApiVersion.V_8).build() - ); + return List.of(new Route(POST, "/_query")); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 11429971b57bb..903c0f948f2e1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.Row; @@ -29,8 +30,10 @@ import org.elasticsearch.xpack.ql.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; +import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.Project; +import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; @@ -137,6 +140,7 @@ else if (p.resolved()) { checkOperationsOnUnsignedLong(p, failures); checkBinaryComparison(p, failures); }); + checkRemoteEnrich(plan, failures); // gather metrics if (failures.isEmpty()) { @@ -376,4 +380,46 @@ private static Failure validateUnsignedLongNegation(Neg neg) { } return null; } + + /** + * Ensure that no remote enrich is allowed after a reduction or an enrich with coordinator mode. + *

    + * TODO: + * For Limit and TopN, we can insert the same node after the remote enrich (also needs to move projections around) + * to eliminate this limitation. Otherwise, we force users to write queries that might not perform well. + * For example, `FROM test | ORDER @timestamp | LIMIT 10 | ENRICH[ccq.mode:remote]` doesn't work. + * In that case, users have to write it as `FROM test | ENRICH[ccq.mode:remote] | ORDER @timestamp | LIMIT 10`, + * which is equivalent to bringing all data to the coordinating cluster. + * We might consider implementing the actual remote enrich on the coordinating cluster, however, this requires + * retaining the originating cluster and restructing pages for routing, which might be complicated. + */ + private static void checkRemoteEnrich(LogicalPlan plan, Set failures) { + boolean[] agg = { false }; + boolean[] limit = { false }; + boolean[] enrichCoord = { false }; + + plan.forEachUp(UnaryPlan.class, u -> { + if (u instanceof Limit) { + limit[0] = true; // TODO: Make Limit then enrich_remote work + } + if (u instanceof Aggregate) { + agg[0] = true; + } else if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.COORDINATOR) { + enrichCoord[0] = true; + } + if (u instanceof Enrich enrich && enrich.mode() == Enrich.Mode.REMOTE) { + if (limit[0]) { + failures.add(fail(enrich, "enrich with [ccq.mode:remote] can't be executed after LIMIT")); + } + if (agg[0]) { + failures.add(fail(enrich, "enrich with [ccq.mode:remote] can't be executed after STATS")); + } + if (enrichCoord[0]) { + failures.add( + fail(enrich, "enrich with [ccq.mode:remote] can't be executed after another enrich with [ccq.mode:coordinator]") + ); + } + } + }); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index ef708a039ce75..325b0677fc72c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -21,8 +21,6 @@ import org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer; -import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.plan.physical.EnrichExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.EsSourceExec; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; @@ -74,19 +72,6 @@ public static Tuple breakPlanBetweenCoordinatorAndDa return new Tuple<>(coordinatorPlan, dataNodePlan.get()); } - public static boolean hasUnsupportedEnrich(PhysicalPlan plan) { - boolean[] found = { false }; - plan.forEachDown(p -> { - if (p instanceof EnrichExec enrich && enrich.mode() == Enrich.Mode.REMOTE) { - found[0] = true; - } - if (p instanceof FragmentExec f) { - f.fragment().forEachDown(Enrich.class, e -> found[0] |= e.mode() == Enrich.Mode.REMOTE); - } - }); - return found[0]; - } - /** * Returns a set of concrete indices after resolving the original indices specified in the FROM command. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 9428d21e4cdbd..b747026dcbfb1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -149,13 +149,20 @@ public void execute( PhysicalPlan coordinatorPlan = new OutputExec(coordinatorAndDataNodePlan.v1(), collectedPages::add); PhysicalPlan dataNodePlan = coordinatorAndDataNodePlan.v2(); if (dataNodePlan != null && dataNodePlan instanceof ExchangeSinkExec == false) { - listener.onFailure(new IllegalStateException("expect data node plan starts with an ExchangeSink; got " + dataNodePlan)); + assert false : "expected data node plan starts with an ExchangeSink; got " + dataNodePlan; + listener.onFailure(new IllegalStateException("expected data node plan starts with an ExchangeSink; got " + dataNodePlan)); return; } Map clusterToConcreteIndices = transportService.getRemoteClusterService() .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, PlannerUtils.planConcreteIndices(physicalPlan).toArray(String[]::new)); QueryPragmas queryPragmas = configuration.pragmas(); - if (dataNodePlan == null || clusterToConcreteIndices.values().stream().allMatch(v -> v.indices().length == 0)) { + if (dataNodePlan == null) { + if (clusterToConcreteIndices.values().stream().allMatch(v -> v.indices().length == 0) == false) { + String error = "expected no concrete indices without data node plan; got " + clusterToConcreteIndices; + assert false : error; + listener.onFailure(new IllegalStateException(error)); + return; + } var computeContext = new ComputeContext( sessionId, RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, @@ -171,15 +178,18 @@ public void execute( listener.map(driverProfiles -> new Result(collectedPages, driverProfiles)) ); return; + } else { + if (clusterToConcreteIndices.values().stream().allMatch(v -> v.indices().length == 0)) { + var error = "expected concrete indices with data node plan but got empty; data node plan " + dataNodePlan; + assert false : error; + listener.onFailure(new IllegalStateException(error)); + return; + } } Map clusterToOriginalIndices = transportService.getRemoteClusterService() .groupIndices(SearchRequest.DEFAULT_INDICES_OPTIONS, PlannerUtils.planOriginalIndices(physicalPlan)); var localOriginalIndices = clusterToOriginalIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); var localConcreteIndices = clusterToConcreteIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - if (PlannerUtils.hasUnsupportedEnrich(physicalPlan)) { - listener.onFailure(new IllegalArgumentException("Enrich REMOTE mode is not supported yet")); - return; - } final var responseHeadersCollector = new ResponseHeadersCollector(transportService.getThreadPool().getThreadContext()); listener = ActionListener.runBefore(listener, responseHeadersCollector::finish); final AtomicBoolean cancelled = new AtomicBoolean(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index aba1f5cfd6b40..17dad71401119 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -32,6 +32,7 @@ import org.elasticsearch.compute.operator.exchange.ExchangeSinkOperator; import org.elasticsearch.compute.operator.exchange.ExchangeSourceOperator; import org.elasticsearch.compute.operator.topn.TopNOperatorStatus; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -58,6 +59,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Stream; @@ -144,7 +146,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of( new RestEsqlQueryAction(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index c29e10f37da28..74b640a723d3f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; +import org.elasticsearch.xpack.esql.analysis.VerificationException; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThan; @@ -239,6 +240,17 @@ private static EnrichResolution setupEnrichResolution() { Map.of("department", new EsField("department", DataTypes.KEYWORD, Map.of(), true)) ) ); + enrichResolution.addResolvedPolicy( + "departments", + Enrich.Mode.REMOTE, + new ResolvedEnrichPolicy( + "employee_id", + EnrichPolicy.MATCH_TYPE, + List.of("department"), + Map.of("cluster_1", ".enrich-departments-2"), + Map.of("department", new EsField("department", DataTypes.KEYWORD, Map.of(), true)) + ) + ); enrichResolution.addResolvedPolicy( "supervisors", Enrich.Mode.ANY, @@ -261,6 +273,17 @@ private static EnrichResolution setupEnrichResolution() { Map.of("supervisor", new EsField("supervisor", DataTypes.KEYWORD, Map.of(), true)) ) ); + enrichResolution.addResolvedPolicy( + "supervisors", + Enrich.Mode.REMOTE, + new ResolvedEnrichPolicy( + "department", + EnrichPolicy.MATCH_TYPE, + List.of("supervisor"), + Map.of("cluster_1", ".enrich-supervisors-b"), + Map.of("supervisor", new EsField("supervisor", DataTypes.KEYWORD, Map.of(), true)) + ) + ); return enrichResolution; } @@ -2758,6 +2781,75 @@ public void testEnrichBeforeAggregation() { var eval = as(fragment.fragment(), Eval.class); as(eval.child(), EsRelation.class); } + { + var plan = physicalPlan(""" + from test + | eval employee_id = to_str(emp_no) + | ENRICH[ccq.mode:remote] departments + | STATS size=count(*) BY department"""); + var limit = as(plan, LimitExec.class); + var finalAggs = as(limit.child(), AggregateExec.class); + assertThat(finalAggs.getMode(), equalTo(FINAL)); + var exchange = as(finalAggs.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var partialAggs = as(fragment.fragment(), Aggregate.class); + var enrich = as(partialAggs.child(), Enrich.class); + assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE)); + assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2"))); + var eval = as(enrich.child(), Eval.class); + as(eval.child(), EsRelation.class); + } + } + + public void testEnrichAfterAggregation() { + { + var plan = physicalPlan(""" + from test + | STATS size=count(*) BY emp_no + | eval employee_id = to_str(emp_no) + | ENRICH[ccq.mode:any] departments + """); + var enrich = as(plan, EnrichExec.class); + assertThat(enrich.mode(), equalTo(Enrich.Mode.ANY)); + assertThat(enrich.concreteIndices(), equalTo(Map.of("", ".enrich-departments-1", "cluster_1", ".enrich-departments-2"))); + var eval = as(enrich.child(), EvalExec.class); + var limit = as(eval.child(), LimitExec.class); + var finalAggs = as(limit.child(), AggregateExec.class); + assertThat(finalAggs.getMode(), equalTo(FINAL)); + var exchange = as(finalAggs.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var partialAggs = as(fragment.fragment(), Aggregate.class); + as(partialAggs.child(), EsRelation.class); + } + { + var plan = physicalPlan(""" + from test + | STATS size=count(*) BY emp_no + | eval employee_id = to_str(emp_no) + | ENRICH[ccq.mode:coordinator] departments + """); + var enrich = as(plan, EnrichExec.class); + assertThat(enrich.mode(), equalTo(Enrich.Mode.COORDINATOR)); + assertThat(enrich.concreteIndices(), equalTo(Map.of("", ".enrich-departments-3"))); + var eval = as(enrich.child(), EvalExec.class); + var limit = as(eval.child(), LimitExec.class); + var finalAggs = as(limit.child(), AggregateExec.class); + assertThat(finalAggs.getMode(), equalTo(FINAL)); + var exchange = as(finalAggs.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var partialAggs = as(fragment.fragment(), Aggregate.class); + as(partialAggs.child(), EsRelation.class); + } + } + + public void testAggThenEnrichRemote() { + var error = expectThrows(VerificationException.class, () -> physicalPlan(""" + from test + | STATS size=count(*) BY emp_no + | eval employee_id = to_str(emp_no) + | ENRICH[ccq.mode:remote] departments + """)); + assertThat(error.getMessage(), containsString("line 4:3: enrich with [ccq.mode:remote] can't be executed after STATS")); } public void testEnrichBeforeLimit() { @@ -2793,6 +2885,69 @@ public void testEnrichBeforeLimit() { var partialLimit = as(fragment.fragment(), Limit.class); as(partialLimit.child(), EsRelation.class); } + { + var plan = physicalPlan(""" + FROM test + | EVAL employee_id = to_str(emp_no) + | ENRICH[ccq.mode:remote] departments + | LIMIT 10"""); + var enrich = as(plan, EnrichExec.class); + assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE)); + assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2"))); + var eval = as(enrich.child(), EvalExec.class); + var finalLimit = as(eval.child(), LimitExec.class); + var exchange = as(finalLimit.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var partialLimit = as(fragment.fragment(), Limit.class); + as(partialLimit.child(), EsRelation.class); + } + } + + public void testLimitThenEnrich() { + { + var plan = physicalPlan(""" + FROM test + | LIMIT 10 + | EVAL employee_id = to_str(emp_no) + | ENRICH[ccq.mode:any] departments + """); + var enrich = as(plan, EnrichExec.class); + assertThat(enrich.mode(), equalTo(Enrich.Mode.ANY)); + assertThat(enrich.concreteIndices(), equalTo(Map.of("", ".enrich-departments-1", "cluster_1", ".enrich-departments-2"))); + var eval = as(enrich.child(), EvalExec.class); + var finalLimit = as(eval.child(), LimitExec.class); + var exchange = as(finalLimit.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var partialLimit = as(fragment.fragment(), Limit.class); + as(partialLimit.child(), EsRelation.class); + } + { + var plan = physicalPlan(""" + FROM test + | LIMIT 10 + | EVAL employee_id = to_str(emp_no) + | ENRICH[ccq.mode:coordinator] departments + """); + var enrich = as(plan, EnrichExec.class); + assertThat(enrich.mode(), equalTo(Enrich.Mode.COORDINATOR)); + assertThat(enrich.concreteIndices(), equalTo(Map.of("", ".enrich-departments-3"))); + var eval = as(enrich.child(), EvalExec.class); + var finalLimit = as(eval.child(), LimitExec.class); + var exchange = as(finalLimit.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var partialLimit = as(fragment.fragment(), Limit.class); + as(partialLimit.child(), EsRelation.class); + } + } + + public void testLimitThenEnrichRemote() { + var error = expectThrows(VerificationException.class, () -> physicalPlan(""" + FROM test + | LIMIT 10 + | EVAL employee_id = to_str(emp_no) + | ENRICH[ccq.mode:remote] departments + """)); + assertThat(error.getMessage(), containsString("line 4:3: enrich with [ccq.mode:remote] can't be executed after LIMIT")); } public void testEnrichBeforeTopN() { @@ -2829,6 +2984,23 @@ public void testEnrichBeforeTopN() { var eval = as(fragment.fragment(), Eval.class); as(eval.child(), EsRelation.class); } + { + var plan = physicalPlan(""" + FROM test + | EVAL employee_id = to_str(emp_no) + | ENRICH[ccq.mode:remote] departments + | SORT department + | LIMIT 10"""); + var topN = as(plan, TopNExec.class); + var exchange = as(topN.child(), ExchangeExec.class); + var fragment = as(exchange.child(), FragmentExec.class); + var partialTopN = as(fragment.fragment(), TopN.class); + var enrich = as(partialTopN.child(), Enrich.class); + assertThat(enrich.mode(), equalTo(Enrich.Mode.REMOTE)); + assertThat(enrich.concreteIndices(), equalTo(Map.of("cluster_1", ".enrich-departments-2"))); + var eval = as(enrich.child(), Eval.class); + as(eval.child(), EsRelation.class); + } } public void testEnrichAfterTopN() { @@ -2975,6 +3147,19 @@ public void testManyEnrich() { } } + public void testRejectRemoteEnrichAfterCoordinatorEnrich() { + var error = expectThrows(VerificationException.class, () -> physicalPlan(""" + from test + | eval employee_id = to_str(emp_no) + | ENRICH[ccq.mode:coordinator] departments + | ENRICH[ccq.mode:remote] supervisors + """)); + assertThat( + error.getMessage(), + containsString("enrich with [ccq.mode:remote] can't be executed after another enrich with [ccq.mode:coordinator]") + ); + } + @SuppressWarnings("SameParameterValue") private static void assertFilterCondition( Filter filter, diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 270786b1f82e7..4c45864a2da5a 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.ExecutorNames; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; @@ -62,6 +63,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.xpack.core.ClientHelper.FLEET_ORIGIN; @@ -361,7 +363,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of( new RestGetGlobalCheckpointsAction(), diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java index 8931669c53ce8..05b75fe6b01ca 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.frozen.FrozenEngine; @@ -35,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.function.Predicate; import java.util.function.Supplier; public class FrozenIndices extends Plugin implements ActionPlugin, EnginePlugin { @@ -71,7 +73,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Collections.singletonList(new RestFreezeIndexAction()); } diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java index c64b5ada48c3f..92aa0339d731c 100644 --- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java +++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.license.License; import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.plugins.ActionPlugin; @@ -30,6 +31,7 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -64,7 +66,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (false == enabled) { return emptyList(); diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java index a61e4c4e1c69e..e493c8e61ca58 100644 --- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java +++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -57,6 +58,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; /** @@ -135,7 +137,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (enabled == false) { return List.of(); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java index a6fa7cd3ffbc6..8a7ec329e55c3 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java @@ -394,7 +394,6 @@ public void testILMWaitsForTimeSeriesEndTimeToLapse() throws Exception { }, 30, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103981") public void testRollupNonTSIndex() throws Exception { createIndex(index, alias, false); index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); @@ -404,10 +403,19 @@ public void testRollupNonTSIndex() throws Exception { createNewSingletonPolicy(client(), policy, phaseName, new DownsampleAction(fixedInterval, DownsampleAction.DEFAULT_WAIT_TIMEOUT)); updatePolicy(client(), index, policy); - assertBusy(() -> assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep(phaseName).getKey()))); - String rollupIndex = getRollupIndexName(client(), index, fixedInterval); - assertNull("Rollup index should not have been created", rollupIndex); - assertTrue("Source index should not have been deleted", indexExists(index)); + try { + assertBusy(() -> assertThat(getStepKeyForIndex(client(), index), equalTo(PhaseCompleteStep.finalStep(phaseName).getKey()))); + String rollupIndex = getRollupIndexName(client(), index, fixedInterval); + assertNull("Rollup index should not have been created", rollupIndex); + assertTrue("Source index should not have been deleted", indexExists(index)); + } catch (AssertionError ea) { + logger.warn( + "--> original index name is [{}], rollup index name is NULL, possible explanation: {}", + index, + explainIndex(client(), index) + ); + throw ea; + } } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101428") diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index e013eb1520f29..f41524480e2df 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.index.IndexModule; import org.elasticsearch.license.XPackLicenseState; @@ -94,6 +95,7 @@ import java.util.Collection; import java.util.List; import java.util.function.LongSupplier; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; @@ -255,7 +257,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { List handlers = new ArrayList<>(); diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java index 5ffb4b5df08cc..79db30b4b14e8 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestInferenceServiceExtension.java @@ -13,6 +13,8 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; @@ -26,7 +28,10 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; import java.io.IOException; import java.util.ArrayList; @@ -138,6 +143,26 @@ public void infer( } } + @Override + public void chunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ) { + switch (model.getConfigurations().getTaskType()) { + case ANY, SPARSE_EMBEDDING -> listener.onResponse(makeChunkedResults(input)); + default -> listener.onFailure( + new ElasticsearchStatusException( + TaskType.unsupportedTaskTypeErrorMsg(model.getConfigurations().getTaskType(), name()), + RestStatus.BAD_REQUEST + ) + ); + } + } + private SparseEmbeddingResults makeResults(List input) { var embeddings = new ArrayList(); for (int i = 0; i < input.size(); i++) { @@ -150,6 +175,18 @@ private SparseEmbeddingResults makeResults(List input) { return new SparseEmbeddingResults(embeddings); } + private ChunkedSparseEmbeddingResults makeChunkedResults(List input) { + var chunks = new ArrayList(); + for (int i = 0; i < input.size(); i++) { + var tokens = new ArrayList(); + for (int j = 0; j < 5; j++) { + tokens.add(new TextExpansionResults.WeightedToken(Integer.toString(j), (float) j)); + } + chunks.add(new ChunkedTextExpansionResults.ChunkedResult(input.get(i), tokens)); + } + return new ChunkedSparseEmbeddingResults(chunks); + } + @Override public void start(Model model, ActionListener listener) { listener.onResponse(true); diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryImplIT.java similarity index 86% rename from x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java rename to x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryImplIT.java index d6d0eb0bbbf21..614ebee99ae4f 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryImplIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.inference.InferencePlugin; -import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.registry.ModelRegistryImpl; import org.elasticsearch.xpack.inference.services.elser.ElserMlNodeModel; import org.elasticsearch.xpack.inference.services.elser.ElserMlNodeService; import org.elasticsearch.xpack.inference.services.elser.ElserMlNodeServiceSettingsTests; @@ -54,13 +54,13 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; -public class ModelRegistryIT extends ESSingleNodeTestCase { +public class ModelRegistryImplIT extends ESSingleNodeTestCase { - private ModelRegistry modelRegistry; + private ModelRegistryImpl ModelRegistryImpl; @Before public void createComponents() { - modelRegistry = new ModelRegistry(client()); + ModelRegistryImpl = new ModelRegistryImpl(client()); } @Override @@ -74,7 +74,7 @@ public void testStoreModel() throws Exception { AtomicReference storeModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.storeModel(model, listener), storeModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), storeModelHolder, exceptionHolder); assertThat(storeModelHolder.get(), is(true)); assertThat(exceptionHolder.get(), is(nullValue())); @@ -86,7 +86,7 @@ public void testStoreModelWithUnknownFields() throws Exception { AtomicReference storeModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.storeModel(model, listener), storeModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), storeModelHolder, exceptionHolder); assertNull(storeModelHolder.get()); assertNotNull(exceptionHolder.get()); @@ -105,12 +105,12 @@ public void testGetModel() throws Exception { AtomicReference putModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); // now get the model - AtomicReference modelHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.getModelWithSecrets(inferenceEntityId, listener), modelHolder, exceptionHolder); + AtomicReference modelHolder = new AtomicReference<>(); + blockingCall(listener -> ModelRegistryImpl.getModelWithSecrets(inferenceEntityId, listener), modelHolder, exceptionHolder); assertThat(exceptionHolder.get(), is(nullValue())); assertThat(modelHolder.get(), not(nullValue())); @@ -132,13 +132,13 @@ public void testStoreModelFailsWhenModelExists() throws Exception { AtomicReference putModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); assertThat(exceptionHolder.get(), is(nullValue())); putModelHolder.set(false); // an model with the same id exists - blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(false)); assertThat(exceptionHolder.get(), not(nullValue())); assertThat( @@ -153,20 +153,20 @@ public void testDeleteModel() throws Exception { Model model = buildElserModelConfig(id, TaskType.SPARSE_EMBEDDING); AtomicReference putModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); } AtomicReference deleteResponseHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.deleteModel("model1", listener), deleteResponseHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.deleteModel("model1", listener), deleteResponseHolder, exceptionHolder); assertThat(exceptionHolder.get(), is(nullValue())); assertTrue(deleteResponseHolder.get()); // get should fail deleteResponseHolder.set(false); - AtomicReference modelHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.getModelWithSecrets("model1", listener), modelHolder, exceptionHolder); + AtomicReference modelHolder = new AtomicReference<>(); + blockingCall(listener -> ModelRegistryImpl.getModelWithSecrets("model1", listener), modelHolder, exceptionHolder); assertThat(exceptionHolder.get(), not(nullValue())); assertFalse(deleteResponseHolder.get()); @@ -186,13 +186,13 @@ public void testGetModelsByTaskType() throws InterruptedException { AtomicReference putModelHolder = new AtomicReference<>(); AtomicReference exceptionHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); } AtomicReference exceptionHolder = new AtomicReference<>(); - AtomicReference> modelHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.getModelsByTaskType(TaskType.SPARSE_EMBEDDING, listener), modelHolder, exceptionHolder); + AtomicReference> modelHolder = new AtomicReference<>(); + blockingCall(listener -> ModelRegistryImpl.getModelsByTaskType(TaskType.SPARSE_EMBEDDING, listener), modelHolder, exceptionHolder); assertThat(modelHolder.get(), hasSize(3)); var sparseIds = sparseAndTextEmbeddingModels.stream() .filter(m -> m.getConfigurations().getTaskType() == TaskType.SPARSE_EMBEDDING) @@ -203,7 +203,7 @@ public void testGetModelsByTaskType() throws InterruptedException { assertThat(m.secrets().keySet(), empty()); }); - blockingCall(listener -> modelRegistry.getModelsByTaskType(TaskType.TEXT_EMBEDDING, listener), modelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.getModelsByTaskType(TaskType.TEXT_EMBEDDING, listener), modelHolder, exceptionHolder); assertThat(modelHolder.get(), hasSize(2)); var denseIds = sparseAndTextEmbeddingModels.stream() .filter(m -> m.getConfigurations().getTaskType() == TaskType.TEXT_EMBEDDING) @@ -227,13 +227,13 @@ public void testGetAllModels() throws InterruptedException { var model = createModel(randomAlphaOfLength(5), randomFrom(TaskType.values()), service); createdModels.add(model); - blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(model, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); assertNull(exceptionHolder.get()); } - AtomicReference> modelHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder); + AtomicReference> modelHolder = new AtomicReference<>(); + blockingCall(listener -> ModelRegistryImpl.getAllModels(listener), modelHolder, exceptionHolder); assertThat(modelHolder.get(), hasSize(modelCount)); var getAllModels = modelHolder.get(); @@ -257,18 +257,18 @@ public void testGetModelWithSecrets() throws InterruptedException { AtomicReference exceptionHolder = new AtomicReference<>(); var modelWithSecrets = createModelWithSecrets(inferenceEntityId, randomFrom(TaskType.values()), service, secret); - blockingCall(listener -> modelRegistry.storeModel(modelWithSecrets, listener), putModelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.storeModel(modelWithSecrets, listener), putModelHolder, exceptionHolder); assertThat(putModelHolder.get(), is(true)); assertNull(exceptionHolder.get()); - AtomicReference modelHolder = new AtomicReference<>(); - blockingCall(listener -> modelRegistry.getModelWithSecrets(inferenceEntityId, listener), modelHolder, exceptionHolder); + AtomicReference modelHolder = new AtomicReference<>(); + blockingCall(listener -> ModelRegistryImpl.getModelWithSecrets(inferenceEntityId, listener), modelHolder, exceptionHolder); assertThat(modelHolder.get().secrets().keySet(), hasSize(1)); var secretSettings = (Map) modelHolder.get().secrets().get("secret_settings"); assertThat(secretSettings.get("secret"), equalTo(secret)); // get model without secrets - blockingCall(listener -> modelRegistry.getModel(inferenceEntityId, listener), modelHolder, exceptionHolder); + blockingCall(listener -> ModelRegistryImpl.getModel(inferenceEntityId, listener), modelHolder, exceptionHolder); assertThat(modelHolder.get().secrets().keySet(), empty()); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index c23e245b5696c..efde4f28a27e1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -14,6 +14,8 @@ import org.elasticsearch.inference.SecretSettings; import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.LegacyTextEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; @@ -57,6 +59,23 @@ public static List getNamedWriteables() { new NamedWriteableRegistry.Entry(InferenceServiceResults.class, TextEmbeddingByteResults.NAME, TextEmbeddingByteResults::new) ); + // Chunked inference results + namedWriteables.add( + new NamedWriteableRegistry.Entry( + InferenceServiceResults.class, + ChunkedSparseEmbeddingResults.NAME, + ChunkedSparseEmbeddingResults::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + InferenceServiceResults.class, + ChunkedTextEmbeddingResults.NAME, + ChunkedTextEmbeddingResults::new + ) + ); + // TODO add text embedding byte result + // Empty default task settings namedWriteables.add(new NamedWriteableRegistry.Entry(TaskSettings.class, EmptyTaskSettings.NAME, EmptyTaskSettings::new)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index ea62ca8620bf5..905a92e899784 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -20,11 +20,15 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.InferenceServiceRegistryImpl; +import org.elasticsearch.inference.ModelRegistry; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ExtensiblePlugin; +import org.elasticsearch.plugins.InferenceRegistryPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; @@ -47,8 +51,9 @@ import org.elasticsearch.xpack.inference.external.http.HttpSettings; import org.elasticsearch.xpack.inference.external.http.retry.RetrySettings; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderFactory; +import org.elasticsearch.xpack.inference.external.http.sender.RequestExecutorServiceSettings; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; -import org.elasticsearch.xpack.inference.registry.ModelRegistry; +import org.elasticsearch.xpack.inference.registry.ModelRegistryImpl; import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceModelAction; import org.elasticsearch.xpack.inference.rest.RestGetInferenceModelAction; import org.elasticsearch.xpack.inference.rest.RestInferenceAction; @@ -63,11 +68,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; -public class InferencePlugin extends Plugin implements ActionPlugin, ExtensiblePlugin, SystemIndexPlugin { +public class InferencePlugin extends Plugin implements ActionPlugin, ExtensiblePlugin, SystemIndexPlugin, InferenceRegistryPlugin { public static final String NAME = "inference"; public static final String UTILITY_THREAD_POOL_NAME = "inference_utility"; @@ -76,6 +82,8 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP private final SetOnce serviceComponents = new SetOnce<>(); private final SetOnce inferenceServiceRegistry = new SetOnce<>(); + private final SetOnce modelRegistry = new SetOnce<>(); + private List inferenceServiceExtensions; public InferencePlugin(Settings settings) { @@ -102,7 +110,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of( new RestInferenceAction(), @@ -126,7 +135,7 @@ public Collection createComponents(PluginServices services) { ); httpFactory.set(httpRequestSenderFactory); - ModelRegistry modelRegistry = new ModelRegistry(services.client()); + ModelRegistry modelReg = new ModelRegistryImpl(services.client()); if (inferenceServiceExtensions == null) { inferenceServiceExtensions = new ArrayList<>(); @@ -135,11 +144,13 @@ public Collection createComponents(PluginServices services) { inferenceServices.add(this::getInferenceServiceFactories); var factoryContext = new InferenceServiceExtension.InferenceServiceFactoryContext(services.client()); - var registry = new InferenceServiceRegistry(inferenceServices, factoryContext); - registry.init(services.client()); - inferenceServiceRegistry.set(registry); + var inferenceRegistry = new InferenceServiceRegistryImpl(inferenceServices, factoryContext); + inferenceRegistry.init(services.client()); + inferenceServiceRegistry.set(inferenceRegistry); + modelRegistry.set(modelReg); - return List.of(modelRegistry, registry); + // Don't return components as they will be registered using InferenceRegistryPlugin methods to retrieve them + return List.of(); } @Override @@ -213,7 +224,8 @@ public List> getSettings() { HttpRequestSenderFactory.HttpRequestSender.getSettings(), ThrottlerManager.getSettings(), RetrySettings.getSettingsDefinitions(), - Truncator.getSettings() + Truncator.getSettings(), + RequestExecutorServiceSettings.getSettingsDefinitions() ).flatMap(Collection::stream).collect(Collectors.toList()); } @@ -234,4 +246,14 @@ public void close() { IOUtils.closeWhileHandlingException(inferenceServiceRegistry.get(), throttlerToClose); } + + @Override + public InferenceServiceRegistry getInferenceServiceRegistry() { + return inferenceServiceRegistry.get(); + } + + @Override + public ModelRegistry getModelRegistry() { + return modelRegistry.get(); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java index 6a5a3f5a137e1..9b110f7b8e7a4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java @@ -23,12 +23,12 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.ModelRegistry; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction; -import org.elasticsearch.xpack.inference.registry.ModelRegistry; public class TransportDeleteInferenceModelAction extends AcknowledgedTransportMasterNodeAction { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java index 2de1aecea118c..0f7e48c4f8140 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelRegistry; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -24,7 +25,6 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction; import org.elasticsearch.xpack.inference.InferencePlugin; -import org.elasticsearch.xpack.inference.registry.ModelRegistry; import java.util.ArrayList; import java.util.List; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index fb3974fc12e8b..ece4fee1c935f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -16,11 +16,11 @@ import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelRegistry; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.inference.registry.ModelRegistry; public class TransportInferenceAction extends HandledTransportAction { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java index f0ff07f79e959..f94da64558132 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelRegistry; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -42,7 +43,6 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.MlPlatformArchitecturesUtil; import org.elasticsearch.xpack.inference.InferencePlugin; -import org.elasticsearch.xpack.inference.registry.ModelRegistry; import java.io.IOException; import java.util.Map; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueue.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueue.java new file mode 100644 index 0000000000000..e73151b44a3e4 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueue.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.common; + +import org.elasticsearch.core.Nullable; + +import java.util.Collection; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Provides a limited functionality queue that can have its capacity adjusted. + * @param the items to store in the queue + */ +public class AdjustableCapacityBlockingQueue { + + private BlockingQueue currentQueue; + /** + * When the capacity of the {@link AdjustableCapacityBlockingQueue#currentQueue} changes, any items that can't fit in the new queue + * will be placed in this secondary queue (the items from the front of the queue first). Then when an operation occurs to remove an + * item from the queue we'll attempt to grab it from this secondary queue first. That way we guarantee that items that were in the + * old queue will be read first. + */ + private final BlockingQueue prioritizedReadingQueue; + private final QueueCreator queueCreator; + private final ReentrantReadWriteLock lock; + + /** + * Constructs the adjustable capacity queue + * @param queueCreator a {@link QueueCreator} object for handling how to create the {@link BlockingQueue} + * @param initialCapacity the initial capacity of the queue, if null the queue will be unbounded + */ + public AdjustableCapacityBlockingQueue(QueueCreator queueCreator, @Nullable Integer initialCapacity) { + this.queueCreator = Objects.requireNonNull(queueCreator); + currentQueue = createCurrentQueue(queueCreator, initialCapacity); + lock = new ReentrantReadWriteLock(); + prioritizedReadingQueue = queueCreator.create(); + } + + private static BlockingQueue createCurrentQueue(QueueCreator queueCreator, @Nullable Integer initialCapacity) { + if (initialCapacity == null) { + return queueCreator.create(); + } + + return queueCreator.create(initialCapacity); + } + + /** + * Sets the capacity of the queue. If the new capacity is smaller than the current number of elements in the queue, the + * elements that exceed the new capacity are retained. In this situation the {@link AdjustableCapacityBlockingQueue#size()} method + * could return a value greater than the specified capacity. + *
    + * This is potentially an expensive operation because a new internal queue is instantiated. + * @param newCapacity the new capacity for the queue + */ + public void setCapacity(int newCapacity) { + final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock(); + + writeLock.lock(); + try { + BlockingQueue newQueue = queueCreator.create(newCapacity); + // Drain the first items from the queue, so they will get read first. + // Only drain the amount that wouldn't fit in the new queue + // If the new capacity is larger than the current queue size then we don't need to drain any + // they will all fit within the newly created queue. In this situation the queue size - capacity + // would result in a negative value which is ignored + if (currentQueue.size() > newCapacity) { + currentQueue.drainTo(prioritizedReadingQueue, currentQueue.size() - newCapacity); + } + currentQueue.drainTo(newQueue, newCapacity); + currentQueue = newQueue; + } finally { + writeLock.unlock(); + } + } + + public boolean offer(E item) { + final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + + readLock.lock(); + try { + return currentQueue.offer(item); + } finally { + readLock.unlock(); + } + } + + public int drainTo(Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + public int drainTo(Collection c, int maxElements) { + final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + + readLock.lock(); + try { + var numberOfDrainedOldItems = prioritizedReadingQueue.drainTo(c, maxElements); + var numberOfDrainedCurrentItems = currentQueue.drainTo(c, maxElements - numberOfDrainedOldItems); + + return numberOfDrainedCurrentItems + numberOfDrainedOldItems; + } finally { + readLock.unlock(); + } + } + + public E poll(long timeout, TimeUnit timeUnit) throws InterruptedException { + final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + + readLock.lockInterruptibly(); + try { + // no new items should be added to the old queue, so we shouldn't need to wait on it + var oldItem = prioritizedReadingQueue.poll(); + + if (oldItem != null) { + return oldItem; + } + + return currentQueue.poll(timeout, timeUnit); + } finally { + readLock.unlock(); + } + } + + public E take() throws InterruptedException { + final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + + readLock.lockInterruptibly(); + try { + var oldItem = prioritizedReadingQueue.poll(); + + if (oldItem != null) { + return oldItem; + } + + return currentQueue.take(); + } finally { + readLock.unlock(); + } + } + + /** + * Returns the number of elements stored in the queue. If the capacity was recently changed, the value returned could be + * greater than the capacity. This occurs when the capacity was reduced and there were more elements in the queue than the + * new capacity. + * @return the number of elements in the queue. + */ + public int size() { + return currentQueue.size() + prioritizedReadingQueue.size(); + } + + /** + * The number of additional elements that his queue can accept without blocking. + */ + public int remainingCapacity() { + return currentQueue.remainingCapacity(); + } + + /** + * Provides a contract for creating a {@link BlockingQueue} + * @param items to store in the queue + */ + public interface QueueCreator { + + /** + * Creates a new {@link BlockingQueue} with the specified capacity. + * @param capacity the number of items that can be stored in the queue + * @return a new {@link BlockingQueue} + */ + BlockingQueue create(int capacity); + + /** + * Creates a new {@link BlockingQueue} with an unbounded capacity. + * @return a new {@link BlockingQueue} + */ + BlockingQueue create(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/RequestExecutor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/RequestExecutor.java new file mode 100644 index 0000000000000..5c8fa62ba88f9 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/RequestExecutor.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xpack.inference.external.request.HttpRequest; + +import java.util.concurrent.TimeUnit; + +public interface RequestExecutor { + void start(); + + void shutdown(); + + boolean isShutdown(); + + boolean isTerminated(); + + boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException; + + void execute(HttpRequest request, @Nullable TimeValue timeout, ActionListener listener); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderFactory.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderFactory.java index edceb8324fbc9..c773f57933415 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderFactory.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderFactory.java @@ -78,7 +78,7 @@ public static final class HttpRequestSender implements Sender { private final ThreadPool threadPool; private final HttpClientManager manager; - private final HttpRequestExecutorService service; + private final RequestExecutorService service; private final AtomicBoolean started = new AtomicBoolean(false); private volatile TimeValue maxRequestTimeout; private final CountDownLatch startCompleted = new CountDownLatch(2); @@ -92,7 +92,13 @@ private HttpRequestSender( ) { this.threadPool = Objects.requireNonNull(threadPool); this.manager = Objects.requireNonNull(httpClientManager); - service = new HttpRequestExecutorService(serviceName, manager.getHttpClient(), threadPool, startCompleted); + service = new RequestExecutorService( + serviceName, + manager.getHttpClient(), + threadPool, + startCompleted, + new RequestExecutorServiceSettings(settings, clusterService) + ); this.maxRequestTimeout = MAX_REQUEST_TIMEOUT.get(settings); addSettingsUpdateConsumers(clusterService); @@ -138,7 +144,7 @@ public void close() throws IOException { public void send(HttpRequest request, @Nullable TimeValue timeout, ActionListener listener) { assert started.get() : "call start() before sending a request"; waitForStartToComplete(); - service.send(request, timeout, listener); + service.execute(request, timeout, listener); } private void waitForStartToComplete() { @@ -159,7 +165,7 @@ private void waitForStartToComplete() { public void send(HttpRequest request, ActionListener listener) { assert started.get() : "call start() before sending a request"; waitForStartToComplete(); - service.send(request, maxRequestTimeout, listener); + service.execute(request, maxRequestTimeout, listener); } public static List> getSettings() { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpTask.java deleted file mode 100644 index 6881d75524bda..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpTask.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.http.sender; - -import org.elasticsearch.common.util.concurrent.AbstractRunnable; - -abstract class HttpTask extends AbstractRunnable { - public boolean shouldShutdown() { - return false; - } -} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ShutdownTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java similarity index 78% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ShutdownTask.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java index 9ec2edf514e80..c5e533eb7d8fe 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ShutdownTask.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java @@ -7,11 +7,9 @@ package org.elasticsearch.xpack.inference.external.http.sender; -class ShutdownTask extends HttpTask { - @Override - public boolean shouldShutdown() { - return true; - } +import org.elasticsearch.common.util.concurrent.AbstractRunnable; + +class NoopTask extends AbstractRunnable { @Override public void onFailure(Exception e) {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestExecutorService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java similarity index 62% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestExecutorService.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java index 84aac7cde6bf5..47b4d49b8f46e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestExecutorService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java @@ -11,29 +11,27 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.common.AdjustableCapacityBlockingQueue; import org.elasticsearch.xpack.inference.external.http.HttpClient; import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.RequestExecutor; import org.elasticsearch.xpack.inference.external.request.HttpRequest; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import static org.elasticsearch.core.Strings.format; @@ -49,51 +47,103 @@ * attempting to execute a task (aka waiting for the connection manager to lease a connection). See * {@link org.apache.http.client.config.RequestConfig.Builder#setConnectionRequestTimeout} for more info. */ -class HttpRequestExecutorService implements ExecutorService { - private static final Logger logger = LogManager.getLogger(HttpRequestExecutorService.class); +class RequestExecutorService implements RequestExecutor { + + private static final AdjustableCapacityBlockingQueue.QueueCreator QUEUE_CREATOR = + new AdjustableCapacityBlockingQueue.QueueCreator<>() { + @Override + public BlockingQueue create(int capacity) { + BlockingQueue queue; + if (capacity <= 0) { + queue = create(); + } else { + queue = new LinkedBlockingQueue<>(capacity); + } + + return queue; + } + + @Override + public BlockingQueue create() { + return new LinkedBlockingQueue<>(); + } + }; + private static final Logger logger = LogManager.getLogger(RequestExecutorService.class); private final String serviceName; - private final BlockingQueue queue; + private final AdjustableCapacityBlockingQueue queue; private final AtomicBoolean running = new AtomicBoolean(true); private final CountDownLatch terminationLatch = new CountDownLatch(1); private final HttpClientContext httpContext; private final HttpClient httpClient; private final ThreadPool threadPool; private final CountDownLatch startupLatch; + private final BlockingQueue controlQueue = new LinkedBlockingQueue<>(); - @SuppressForbidden(reason = "wraps a queue and handles errors appropriately") - HttpRequestExecutorService(String serviceName, HttpClient httpClient, ThreadPool threadPool, @Nullable CountDownLatch startupLatch) { - this(serviceName, httpClient, threadPool, new LinkedBlockingQueue<>(), startupLatch); - } - - @SuppressForbidden(reason = "wraps a queue and handles errors appropriately") - HttpRequestExecutorService( + RequestExecutorService( String serviceName, HttpClient httpClient, ThreadPool threadPool, - int capacity, - @Nullable CountDownLatch startupLatch + @Nullable CountDownLatch startupLatch, + RequestExecutorServiceSettings settings ) { - this(serviceName, httpClient, threadPool, new LinkedBlockingQueue<>(capacity), startupLatch); + this(serviceName, httpClient, threadPool, QUEUE_CREATOR, startupLatch, settings); + } + + private static BlockingQueue buildQueue(int capacity) { + BlockingQueue queue; + if (capacity <= 0) { + queue = new LinkedBlockingQueue<>(); + } else { + queue = new LinkedBlockingQueue<>(capacity); + } + + return queue; } /** * This constructor should only be used directly for testing. */ - @SuppressForbidden(reason = "wraps a queue and handles errors appropriately") - HttpRequestExecutorService( + RequestExecutorService( String serviceName, HttpClient httpClient, ThreadPool threadPool, - BlockingQueue queue, - @Nullable CountDownLatch startupLatch + AdjustableCapacityBlockingQueue.QueueCreator createQueue, + @Nullable CountDownLatch startupLatch, + RequestExecutorServiceSettings settings ) { this.serviceName = Objects.requireNonNull(serviceName); this.httpClient = Objects.requireNonNull(httpClient); this.threadPool = Objects.requireNonNull(threadPool); this.httpContext = HttpClientContext.create(); - this.queue = queue; + this.queue = new AdjustableCapacityBlockingQueue<>(createQueue, settings.getQueueCapacity()); this.startupLatch = startupLatch; + + Objects.requireNonNull(settings); + settings.registerQueueCapacityCallback(this::onCapacityChange); + } + + private void onCapacityChange(int capacity) { + logger.debug(() -> Strings.format("Setting queue capacity to [%s]", capacity)); + + var enqueuedCapacityCommand = controlQueue.offer(() -> updateCapacity(capacity)); + if (enqueuedCapacityCommand == false) { + logger.warn("Failed to change request batching service queue capacity. Control queue was full, please try again later."); + } else { + // ensure that the task execution loop wakes up + queue.offer(new NoopTask()); + } + } + + private void updateCapacity(int newCapacity) { + try { + queue.setCapacity(newCapacity); + } catch (Exception e) { + logger.warn( + format("Failed to set the capacity of the task queue to [%s] for request batching service [%s]", newCapacity, serviceName), + e + ); + } } /** @@ -125,13 +175,18 @@ private void signalStartInitiated() { * Protects the task retrieval logic from an unexpected exception. * * @throws InterruptedException rethrows the exception if it occurred retrieving a task because the thread is likely attempting to - * shut down + * shut down */ private void handleTasks() throws InterruptedException { try { - HttpTask task = queue.take(); - if (task.shouldShutdown() || running.get() == false) { - running.set(false); + AbstractRunnable task = queue.take(); + + var command = controlQueue.poll(); + if (command != null) { + command.run(); + } + + if (running.get() == false) { logger.debug(() -> format("Http executor service [%s] exiting", serviceName)); } else { executeTask(task); @@ -143,7 +198,7 @@ private void handleTasks() throws InterruptedException { } } - private void executeTask(HttpTask task) { + private void executeTask(AbstractRunnable task) { try { task.run(); } catch (Exception e) { @@ -155,18 +210,16 @@ private synchronized void notifyRequestsOfShutdown() { assert isShutdown() : "Requests should only be notified if the executor is shutting down"; try { - List notExecuted = new ArrayList<>(); + List notExecuted = new ArrayList<>(); queue.drainTo(notExecuted); - for (HttpTask task : notExecuted) { - rejectTask(task); - } + rejectTasks(notExecuted, this::rejectTaskBecauseOfShutdown); } catch (Exception e) { logger.warn(format("Failed to notify tasks of queuing service [%s] shutdown", serviceName)); } } - private void rejectTask(HttpTask task) { + private void rejectTaskBecauseOfShutdown(AbstractRunnable task) { try { task.onRejection( new EsRejectedExecutionException( @@ -181,6 +234,12 @@ private void rejectTask(HttpTask task) { } } + private void rejectTasks(List tasks, Consumer rejectionFunction) { + for (var task : tasks) { + rejectionFunction.accept(task); + } + } + public int queueSize() { return queue.size(); } @@ -189,16 +248,10 @@ public int queueSize() { public void shutdown() { if (running.compareAndSet(true, false)) { // if this fails because the queue is full, that's ok, we just want to ensure that queue.take() returns - queue.offer(new ShutdownTask()); + queue.offer(new NoopTask()); } } - @Override - public List shutdownNow() { - shutdown(); - return new ArrayList<>(queue); - } - @Override public boolean isShutdown() { return running.get() == false; @@ -216,13 +269,14 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE /** * Send the request at some point in the future. - * @param request the http request to send - * @param timeout the maximum time to wait for this request to complete (failing or succeeding). Once the time elapses, the - * listener::onFailure is called with a {@link org.elasticsearch.ElasticsearchTimeoutException}. - * If null, then the request will wait forever + * + * @param request the http request to send + * @param timeout the maximum time to wait for this request to complete (failing or succeeding). Once the time elapses, the + * listener::onFailure is called with a {@link org.elasticsearch.ElasticsearchTimeoutException}. + * If null, then the request will wait forever * @param listener an {@link ActionListener} for the response or failure */ - public void send(HttpRequest request, @Nullable TimeValue timeout, ActionListener listener) { + public void execute(HttpRequest request, @Nullable TimeValue timeout, ActionListener listener) { RequestTask task = new RequestTask(request, httpClient, httpContext, timeout, threadPool, listener); if (isShutdown()) { @@ -251,69 +305,8 @@ public void send(HttpRequest request, @Nullable TimeValue timeout, ActionListene } } - /** - * This method is not supported. Use {@link #send} instead. - * @param runnable the runnable task - */ - @Override - public void execute(Runnable runnable) { - throw new UnsupportedOperationException("use send instead"); - } - - /** - * This method is not supported. Use {@link #send} instead. - */ - @Override - public Future submit(Callable task) { - throw new UnsupportedOperationException("use send instead"); - } - - /** - * This method is not supported. Use {@link #send} instead. - */ - @Override - public Future submit(Runnable task, T result) { - throw new UnsupportedOperationException("use send instead"); - } - - /** - * This method is not supported. Use {@link #send} instead. - */ - @Override - public Future submit(Runnable task) { - throw new UnsupportedOperationException("use send instead"); - } - - /** - * This method is not supported. Use {@link #send} instead. - */ - @Override - public List> invokeAll(Collection> tasks) throws InterruptedException { - throw new UnsupportedOperationException("use send instead"); - } - - /** - * This method is not supported. Use {@link #send} instead. - */ - @Override - public List> invokeAll(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException { - throw new UnsupportedOperationException("use send instead"); - } - - /** - * This method is not supported. Use {@link #send} instead. - */ - @Override - public T invokeAny(Collection> tasks) throws InterruptedException, ExecutionException { - throw new UnsupportedOperationException("use send instead"); - } - - /** - * This method is not supported. Use {@link #send} instead. - */ - @Override - public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException, - ExecutionException, TimeoutException { - throw new UnsupportedOperationException("use send instead"); + // default for testing + int remainingQueueCapacity() { + return queue.remainingCapacity(); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettings.java new file mode 100644 index 0000000000000..86825035f2d05 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettings.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +public class RequestExecutorServiceSettings { + + /** + * The capacity of the internal queue. Zero is considered unlimited. If a positive value is used, the queue will reject entries + * once it is full. + */ + static final Setting TASK_QUEUE_CAPACITY_SETTING = Setting.intSetting( + "xpack.inference.http.request_executor.queue_capacity", + 2000, + 0, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static List> getSettingsDefinitions() { + return List.of(TASK_QUEUE_CAPACITY_SETTING); + } + + private volatile int queueCapacity; + private final List> queueCapacityCallbacks = new ArrayList>(); + + public RequestExecutorServiceSettings(Settings settings, ClusterService clusterService) { + queueCapacity = TASK_QUEUE_CAPACITY_SETTING.get(settings); + + addSettingsUpdateConsumers(clusterService); + } + + private void addSettingsUpdateConsumers(ClusterService clusterService) { + clusterService.getClusterSettings().addSettingsUpdateConsumer(TASK_QUEUE_CAPACITY_SETTING, this::setQueueCapacity); + } + + // default for testing + void setQueueCapacity(int queueCapacity) { + this.queueCapacity = queueCapacity; + + for (var callback : queueCapacityCallbacks) { + callback.accept(queueCapacity); + } + } + + void registerQueueCapacityCallback(Consumer onChangeCapacityCallback) { + queueCapacityCallbacks.add(onChangeCapacityCallback); + } + + int getQueueCapacity() { + return queueCapacity; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java index 2eefff791b709..cc65d16af652c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java @@ -13,6 +13,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.threadpool.Scheduler; @@ -27,7 +28,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; -class RequestTask extends HttpTask { +class RequestTask extends AbstractRunnable { private static final Logger logger = LogManager.getLogger(RequestTask.class); private static final Scheduler.Cancellable NOOP_TIMEOUT_HANDLER = createDefaultHandler(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistryImpl.java similarity index 86% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistryImpl.java index 0f3aa5b82b189..40921cd38f181 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistryImpl.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -31,6 +32,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelRegistry; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; @@ -55,49 +57,21 @@ import static org.elasticsearch.core.Strings.format; -public class ModelRegistry { +public class ModelRegistryImpl implements ModelRegistry { public record ModelConfigMap(Map config, Map secrets) {} - /** - * Semi parsed model where inference entity id, task type and service - * are known but the settings are not parsed. - */ - public record UnparsedModel( - String inferenceEntityId, - TaskType taskType, - String service, - Map settings, - Map secrets - ) { - - public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap) { - if (modelConfigMap.config() == null) { - throw new ElasticsearchStatusException("Missing config map", RestStatus.BAD_REQUEST); - } - String inferenceEntityId = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.MODEL_ID); - String service = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.SERVICE); - String taskTypeStr = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), TaskType.NAME); - TaskType taskType = TaskType.fromString(taskTypeStr); - - return new UnparsedModel(inferenceEntityId, taskType, service, modelConfigMap.config(), modelConfigMap.secrets()); - } - } - private static final String TASK_TYPE_FIELD = "task_type"; private static final String MODEL_ID_FIELD = "model_id"; - private static final Logger logger = LogManager.getLogger(ModelRegistry.class); + private static final Logger logger = LogManager.getLogger(ModelRegistryImpl.class); private final OriginSettingClient client; - public ModelRegistry(Client client) { + @Inject + public ModelRegistryImpl(Client client) { this.client = new OriginSettingClient(client, ClientHelper.INFERENCE_ORIGIN); } - /** - * Get a model with its secret settings - * @param inferenceEntityId Model to get - * @param listener Model listener - */ + @Override public void getModelWithSecrets(String inferenceEntityId, ActionListener listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { // There should be a hit for the configurations and secrets @@ -106,7 +80,7 @@ public void getModelWithSecrets(String inferenceEntityId, ActionListener listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { // There should be a hit for the configurations and secrets @@ -132,7 +101,7 @@ public void getModel(String inferenceEntityId, ActionListener lis return; } - var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(UnparsedModel::unparsedModelFromMap).toList(); + var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistryImpl::unparsedModelFromMap).toList(); assert modelConfigs.size() == 1; delegate.onResponse(modelConfigs.get(0)); }); @@ -147,12 +116,7 @@ public void getModel(String inferenceEntityId, ActionListener lis client.search(modelSearch, searchListener); } - /** - * Get all models of a particular task type. - * Secret settings are not included - * @param taskType The task type - * @param listener Models listener - */ + @Override public void getModelsByTaskType(TaskType taskType, ActionListener> listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { // Not an error if no models of this task_type @@ -161,7 +125,7 @@ public void getModelsByTaskType(TaskType taskType, ActionListener> listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { // Not an error if no models of this task_type @@ -190,7 +150,7 @@ public void getAllModels(ActionListener> listener) { return; } - var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(UnparsedModel::unparsedModelFromMap).toList(); + var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistryImpl::unparsedModelFromMap).toList(); delegate.onResponse(modelConfigs); }); @@ -257,6 +217,7 @@ private ModelConfigMap createModelConfigMap(SearchHits hits, String inferenceEnt ); } + @Override public void storeModel(Model model, ActionListener listener) { ActionListener bulkResponseActionListener = getStoreModelListener(model, listener); @@ -353,6 +314,7 @@ private static BulkItemResponse.Failure getFirstBulkFailure(BulkResponse bulkRes return null; } + @Override public void deleteModel(String inferenceEntityId, ActionListener listener) { DeleteByQueryRequest request = new DeleteByQueryRequest().setAbortOnVersionConflict(false); request.indices(InferenceIndex.INDEX_PATTERN, InferenceSecretsIndex.INDEX_PATTERN); @@ -377,4 +339,16 @@ private static IndexRequest createIndexRequest(String docId, String indexName, T private QueryBuilder documentIdQuery(String inferenceEntityId) { return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(Model.documentId(inferenceEntityId))); } + + private static UnparsedModel unparsedModelFromMap(ModelRegistryImpl.ModelConfigMap modelConfigMap) { + if (modelConfigMap.config() == null) { + throw new ElasticsearchStatusException("Missing config map", RestStatus.BAD_REQUEST); + } + String modelId = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.MODEL_ID); + String service = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.SERVICE); + String taskTypeStr = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), TaskType.NAME); + TaskType taskType = TaskType.fromString(taskTypeStr); + + return new UnparsedModel(modelId, taskType, service, modelConfigMap.config(), modelConfigMap.secrets()); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java index 0c40863b37db2..16e4626dc9ba3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java @@ -10,6 +10,8 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -54,6 +56,19 @@ public void infer( doInfer(model, input, taskSettings, inputType, listener); } + @Override + public void chunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ) { + init(); + doChunkedInfer(model, input, taskSettings, inputType, chunkingOptions, listener); + } + protected abstract void doInfer( Model model, List input, @@ -62,6 +77,15 @@ protected abstract void doInfer( ActionListener listener ); + protected abstract void doChunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ); + @Override public void start(Model model, ActionListener listener) { init(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 3f608c977f686..bb24faeaff6da 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -13,6 +13,8 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -139,6 +141,18 @@ public void doInfer( action.execute(input, listener); } + @Override + protected void doChunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ) { + listener.onFailure(new ElasticsearchStatusException("Chunking not supported by the {} service", RestStatus.BAD_REQUEST, NAME)); + } + /** * For text embedding models get the embedding size and * update the service settings. diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeService.java index 1d0bd123c69f3..10e7b73a43c28 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeService.java @@ -16,6 +16,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceService; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; @@ -25,6 +28,7 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; @@ -33,7 +37,9 @@ import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; @@ -220,13 +226,10 @@ public void infer( ) { // No task settings to override with requestTaskSettings - if (TaskType.SPARSE_EMBEDDING.isAnyOrSame(model.getConfigurations().getTaskType()) == false) { - listener.onFailure( - new ElasticsearchStatusException( - TaskType.unsupportedTaskTypeErrorMsg(model.getConfigurations().getTaskType(), NAME), - RestStatus.BAD_REQUEST - ) - ); + try { + checkCompatibleTaskType(model.getConfigurations().getTaskType()); + } catch (Exception e) { + listener.onFailure(e); return; } @@ -243,6 +246,47 @@ public void infer( ); } + @Override + public void chunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ) { + try { + checkCompatibleTaskType(model.getConfigurations().getTaskType()); + } catch (Exception e) { + listener.onFailure(e); + return; + } + + var configUpdate = chunkingOptions.settingsArePresent() + ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span()) + : TextExpansionConfigUpdate.EMPTY_UPDATE; + + var request = InferTrainedModelDeploymentAction.Request.forTextInput( + model.getConfigurations().getInferenceEntityId(), + configUpdate, + input, + TimeValue.timeValueSeconds(10) // TODO get timeout from request + ); + request.setChunkResults(true); + + client.execute( + InferTrainedModelDeploymentAction.INSTANCE, + request, + listener.delegateFailureAndWrap((l, inferenceResult) -> l.onResponse(translateChunkedResults(inferenceResult.getResults()))) + ); + } + + private void checkCompatibleTaskType(TaskType taskType) { + if (TaskType.SPARSE_EMBEDDING.isAnyOrSame(taskType) == false) { + throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); + } + } + @Override public void putModel(Model model, ActionListener listener) { if (model instanceof ElserMlNodeModel == false) { @@ -279,6 +323,27 @@ private static ElserMlNodeTaskSettings taskSettingsFromMap(TaskType taskType, Ma return ElserMlNodeTaskSettings.DEFAULT; } + private ChunkedSparseEmbeddingResults translateChunkedResults(List inferenceResults) { + if (inferenceResults.size() != 1) { + throw new ElasticsearchStatusException( + "Expected exactly one chunked sparse embedding result", + RestStatus.INTERNAL_SERVER_ERROR + ); + } + + if (inferenceResults.get(0) instanceof ChunkedTextExpansionResults mlChunkedResult) { + return ChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult); + } else { + throw new ElasticsearchStatusException( + "Expected a chunked inference [{}] received [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + ChunkedTextExpansionResults.NAME, + inferenceResults.get(0).getWriteableName() + ); + } + + } + @Override public String name() { return NAME; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java index dcaa760868c49..5a57699e03c10 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -111,4 +113,17 @@ public void doInfer( var action = huggingFaceModel.accept(actionCreator); action.execute(input, listener); } + + @Override + protected void doChunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ) { + listener.onFailure(new UnsupportedOperationException("Chunked inference not implemented for Hugging Face")); + } + } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 594d7cf2cf31c..d795c75b6c178 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -13,6 +13,8 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -152,6 +154,18 @@ public void doInfer( action.execute(input, listener); } + @Override + protected void doChunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ) { + listener.onFailure(new ElasticsearchStatusException("Chunking not supported by the {} service", RestStatus.BAD_REQUEST, NAME)); + } + /** * For text embedding models get the embedding size and * update the service settings. diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java index 356caecf8fadb..5b7ffb3c8153e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.inference.external.http.HttpSettings; import org.elasticsearch.xpack.inference.external.http.retry.RetrySettings; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderFactory; +import org.elasticsearch.xpack.inference.external.http.sender.RequestExecutorServiceSettings; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import java.util.Collection; @@ -41,7 +42,8 @@ public static ClusterService mockClusterService(Settings settings) { HttpRequestSenderFactory.HttpRequestSender.getSettings(), ThrottlerManager.getSettings(), RetrySettings.getSettingsDefinitions(), - Truncator.getSettings() + Truncator.getSettings(), + RequestExecutorServiceSettings.getSettingsDefinitions() ).flatMap(Collection::stream).collect(Collectors.toSet()); var cSettings = new ClusterSettings(settings, registeredSettings); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueueTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueueTests.java new file mode 100644 index 0000000000000..09cd065ce3cd0 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueueTests.java @@ -0,0 +1,249 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.common; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.hamcrest.Matchers.is; + +public class AdjustableCapacityBlockingQueueTests extends ESTestCase { + private static final AdjustableCapacityBlockingQueue.QueueCreator QUEUE_CREATOR = + new AdjustableCapacityBlockingQueue.QueueCreator<>() { + @Override + public BlockingQueue create(int capacity) { + return new LinkedBlockingQueue<>(capacity); + } + + @Override + public BlockingQueue create() { + return new LinkedBlockingQueue<>(); + } + }; + + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private ThreadPool threadPool; + + @Before + public void init() { + threadPool = createThreadPool(inferenceUtilityPool()); + } + + @After + public void shutdown() { + terminate(threadPool); + } + + public void testSetCapacity_ChangesTheQueueCapacityToTwo() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + assertThat(queue.remainingCapacity(), is(1)); + + queue.setCapacity(2); + assertThat(queue.remainingCapacity(), is(2)); + } + + public void testInitiallySetsCapacityToUnbounded_WhenCapacityIsNull() { + assertThat(new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, null).remainingCapacity(), is(Integer.MAX_VALUE)); + } + + public void testSetCapacity_RemainingCapacityIsZero_WhenReducingTheQueueCapacityToOne_WhenItemsExistInTheQueue() + throws InterruptedException { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 2); + assertThat(queue.remainingCapacity(), is(2)); + assertThat(queue.size(), is(0)); + + queue.offer(0); + queue.offer(1); + assertThat(queue.remainingCapacity(), is(0)); + + queue.setCapacity(1); + assertThat(queue.remainingCapacity(), is(0)); + assertThat(queue.size(), is(2)); + assertThat(queue.take(), is(0)); + assertThat(queue.take(), is(1)); + } + + public void testSetCapacity_RetainsOrdering_WhenReturningItems_AfterDecreasingCapacity() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 3); + assertThat(queue.size(), is(0)); + + queue.offer(0); + queue.offer(1); + queue.offer(2); + assertThat(queue.size(), is(3)); + + queue.setCapacity(2); + + var entriesList = new ArrayList(); + assertThat(queue.drainTo(entriesList), is(3)); + + assertThat(queue.size(), is(0)); + assertThat(entriesList, is(List.of(0, 1, 2))); + } + + public void testSetCapacity_RetainsOrdering_WhenReturningItems_AfterIncreasingCapacity() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 2); + assertThat(queue.size(), is(0)); + + queue.offer(0); + queue.offer(1); + assertThat(queue.size(), is(2)); + + queue.setCapacity(3); + + queue.offer(2); + + var entriesList = new ArrayList(); + assertThat(queue.drainTo(entriesList), is(3)); + + assertThat(queue.size(), is(0)); + assertThat(entriesList, is(List.of(0, 1, 2))); + } + + public void testSetCapacity_RetainsOrdering_WhenReturningItems_AfterDecreasingCapacity_UsingTake() throws InterruptedException { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 3); + assertThat(queue.size(), is(0)); + + queue.offer(0); + queue.offer(1); + queue.offer(2); + assertThat(queue.size(), is(3)); + + queue.setCapacity(2); + + assertThat(queue.take(), is(0)); + assertThat(queue.take(), is(1)); + assertThat(queue.take(), is(2)); + + assertThat(queue.size(), is(0)); + } + + public void testSetCapacity_RetainsOrdering_WhenReturningItems_AfterIncreasingCapacity_UsingTake() throws InterruptedException { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 2); + assertThat(queue.size(), is(0)); + + queue.offer(0); + queue.offer(1); + assertThat(queue.size(), is(2)); + + queue.setCapacity(3); + + queue.offer(2); + + assertThat(queue.take(), is(0)); + assertThat(queue.take(), is(1)); + assertThat(queue.take(), is(2)); + + assertThat(queue.size(), is(0)); + } + + public void testOffer_AddsItemToTheQueue() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + assertThat(queue.size(), is(0)); + + queue.offer(0); + assertThat(queue.size(), is(1)); + } + + public void testDrainTo_MovesAllItemsFromQueueToList() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 2); + assertThat(queue.size(), is(0)); + + queue.offer(0); + queue.offer(1); + assertThat(queue.size(), is(2)); + + var entriesList = new ArrayList(); + queue.drainTo(entriesList); + + assertThat(queue.size(), is(0)); + assertThat(entriesList, is(List.of(0, 1))); + } + + public void testDrainTo_MovesOnlyOneItemFromQueueToList() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 2); + assertThat(queue.size(), is(0)); + + queue.offer(0); + queue.offer(1); + assertThat(queue.size(), is(2)); + + var entriesList = new ArrayList(); + assertThat(queue.drainTo(entriesList, 1), is(1)); + + assertThat(queue.size(), is(1)); + assertThat(entriesList, is(List.of(0))); + } + + public void testPoll_RemovesAnItemFromTheQueue_AfterItBecomesAvailable() throws ExecutionException, InterruptedException, + TimeoutException { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + assertThat(queue.size(), is(0)); + + var waitForOfferCallLatch = new CountDownLatch(1); + + Future pollFuture = threadPool.generic().submit(() -> { + try { + waitForOfferCallLatch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); + return queue.poll(TIMEOUT.getSeconds(), TimeUnit.SECONDS); + } catch (Exception e) { + fail(Strings.format("Failed to polling queue: %s", e)); + } + + return null; + }); + + queue.offer(0); + assertThat(queue.size(), is(1)); + waitForOfferCallLatch.countDown(); + + assertThat(pollFuture.get(TIMEOUT.getSeconds(), TimeUnit.SECONDS), is(0)); + + assertThat(queue.size(), is(0)); + } + + public void testTake_RemovesItemFromQueue() throws InterruptedException { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + assertThat(queue.size(), is(0)); + + queue.offer(0); + assertThat(queue.size(), is(1)); + + assertThat(queue.take(), is(0)); + assertThat(queue.size(), is(0)); + } + + public static AdjustableCapacityBlockingQueue.QueueCreator mockQueueCreator(BlockingQueue backingQueue) { + return new AdjustableCapacityBlockingQueue.QueueCreator<>() { + @Override + public BlockingQueue create(int capacity) { + return backingQueue; + } + + @Override + public BlockingQueue create() { + return backingQueue; + } + }; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictorTests.java index f29120d9026a5..3c263adaddc46 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictorTests.java @@ -10,17 +10,16 @@ import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; import org.apache.http.nio.reactor.IOReactorException; +import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; -import org.junit.After; import org.junit.Before; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doAnswer; @@ -32,16 +31,11 @@ public class IdleConnectionEvictorTests extends ESTestCase { private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); - private ThreadPool threadPool; + private DeterministicTaskQueue taskQueue; @Before public void init() { - threadPool = createThreadPool(inferenceUtilityPool()); - } - - @After - public void shutdown() { - terminate(threadPool); + taskQueue = new DeterministicTaskQueue(); } public void testStart_CallsExecutorSubmit() throws IOReactorException { @@ -87,7 +81,7 @@ public void testCloseExpiredConnections_IsCalled() throws InterruptedException { var manager = mock(PoolingNHttpClientConnectionManager.class); var evictor = new IdleConnectionEvictor( - threadPool, + taskQueue.getThreadPool(), manager, new TimeValue(1, TimeUnit.NANOSECONDS), new TimeValue(1, TimeUnit.NANOSECONDS) @@ -100,7 +94,8 @@ public void testCloseExpiredConnections_IsCalled() throws InterruptedException { return Void.TYPE; }).when(manager).closeExpiredConnections(); - evictor.start(); + startEvictor(evictor); + runLatch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); verify(manager, times(1)).closeExpiredConnections(); @@ -110,7 +105,7 @@ public void testCloseIdleConnections_IsCalled() throws InterruptedException { var manager = mock(PoolingNHttpClientConnectionManager.class); var evictor = new IdleConnectionEvictor( - threadPool, + taskQueue.getThreadPool(), manager, new TimeValue(1, TimeUnit.NANOSECONDS), new TimeValue(1, TimeUnit.NANOSECONDS) @@ -123,7 +118,8 @@ public void testCloseIdleConnections_IsCalled() throws InterruptedException { return Void.TYPE; }).when(manager).closeIdleConnections(anyLong(), any()); - evictor.start(); + startEvictor(evictor); + runLatch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); verify(manager, times(1)).closeIdleConnections(anyLong(), any()); @@ -131,32 +127,38 @@ public void testCloseIdleConnections_IsCalled() throws InterruptedException { public void testIsRunning_ReturnsTrue() throws IOReactorException { var evictor = new IdleConnectionEvictor( - threadPool, + taskQueue.getThreadPool(), createConnectionManager(), new TimeValue(1, TimeUnit.SECONDS), new TimeValue(1, TimeUnit.SECONDS) ); - evictor.start(); + startEvictor(evictor); + assertTrue(evictor.isRunning()); evictor.close(); } public void testIsRunning_ReturnsFalse() throws IOReactorException { var evictor = new IdleConnectionEvictor( - threadPool, + taskQueue.getThreadPool(), createConnectionManager(), new TimeValue(1, TimeUnit.SECONDS), new TimeValue(1, TimeUnit.SECONDS) ); - evictor.start(); + startEvictor(evictor); assertTrue(evictor.isRunning()); evictor.close(); assertFalse(evictor.isRunning()); } + private void startEvictor(IdleConnectionEvictor evictor) { + taskQueue.scheduleNow(evictor::start); + taskQueue.runAllRunnableTasks(); + } + private static PoolingNHttpClientConnectionManager createConnectionManager() throws IOReactorException { return new PoolingNHttpClientConnectionManager(new DefaultConnectingIOReactor()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestExecutorServiceTests.java deleted file mode 100644 index f25312260bfd0..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestExecutorServiceTests.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.external.http.sender; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.inference.external.http.HttpClient; -import org.elasticsearch.xpack.inference.external.http.HttpResult; -import org.elasticsearch.xpack.inference.external.request.HttpRequestTests; -import org.junit.After; -import org.junit.Before; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; -import static org.elasticsearch.xpack.inference.external.http.HttpClientTests.createHttpPost; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class HttpRequestExecutorServiceTests extends ESTestCase { - private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); - private ThreadPool threadPool; - - @Before - public void init() { - threadPool = createThreadPool(inferenceUtilityPool()); - } - - @After - public void shutdown() { - terminate(threadPool); - } - - public void testQueueSize_IsEmpty() { - var service = new HttpRequestExecutorService(getTestName(), mock(HttpClient.class), threadPool, null); - - assertThat(service.queueSize(), is(0)); - } - - public void testQueueSize_IsOne() { - var service = new HttpRequestExecutorService(getTestName(), mock(HttpClient.class), threadPool, null); - service.send(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); - - assertThat(service.queueSize(), is(1)); - } - - public void testExecute_ThrowsUnsupported() { - var service = new HttpRequestExecutorService(getTestName(), mock(HttpClient.class), threadPool, null); - var noopTask = mock(RequestTask.class); - - var thrownException = expectThrows(UnsupportedOperationException.class, () -> service.execute(noopTask)); - assertThat(thrownException.getMessage(), is("use send instead")); - } - - public void testIsTerminated_IsFalse() { - var service = new HttpRequestExecutorService(getTestName(), mock(HttpClient.class), threadPool, null); - - assertFalse(service.isTerminated()); - } - - public void testIsTerminated_IsTrue() throws InterruptedException { - var latch = new CountDownLatch(1); - var service = new HttpRequestExecutorService(getTestName(), mock(HttpClient.class), threadPool, latch); - - service.shutdown(); - service.start(); - latch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); - - assertTrue(service.isTerminated()); - } - - public void testIsTerminated_AfterStopFromSeparateThread() throws Exception { - var waitToShutdown = new CountDownLatch(1); - - var mockHttpClient = mock(HttpClient.class); - doAnswer(invocation -> { - waitToShutdown.countDown(); - return Void.TYPE; - }).when(mockHttpClient).send(any(), any(), any()); - - var service = new HttpRequestExecutorService(getTestName(), mockHttpClient, threadPool, null); - - Future executorTermination = threadPool.generic().submit(() -> { - try { - // wait for a task to be added to be executed before beginning shutdown - waitToShutdown.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); - service.shutdown(); - service.awaitTermination(TIMEOUT.getSeconds(), TimeUnit.SECONDS); - } catch (Exception e) { - fail(Strings.format("Failed to shutdown executor: %s", e)); - } - }); - - PlainActionFuture listener = new PlainActionFuture<>(); - service.send(HttpRequestTests.createMock("inferenceEntityId"), null, listener); - - service.start(); - - try { - executorTermination.get(1, TimeUnit.SECONDS); - } catch (Exception e) { - fail(Strings.format("Executor finished before it was signaled to shutdown: %s", e)); - } - - assertTrue(service.isShutdown()); - assertTrue(service.isTerminated()); - } - - public void testSend_AfterShutdown_Throws() { - var service = new HttpRequestExecutorService("test_service", mock(HttpClient.class), threadPool, null); - - service.shutdown(); - - var listener = new PlainActionFuture(); - service.send(HttpRequestTests.createMock("inferenceEntityId"), null, listener); - - var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); - - assertThat( - thrownException.getMessage(), - is("Failed to enqueue task because the http executor service [test_service] has already shutdown") - ); - } - - public void testSend_Throws_WhenQueueIsFull() { - var service = new HttpRequestExecutorService("test_service", mock(HttpClient.class), threadPool, 1, null); - - service.send(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); - var listener = new PlainActionFuture(); - service.send(HttpRequestTests.createMock("inferenceEntityId"), null, listener); - - var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); - - assertThat( - thrownException.getMessage(), - is("Failed to execute task because the http executor service [test_service] queue is full") - ); - } - - public void testTaskThrowsError_CallsOnFailure() throws Exception { - var httpClient = mock(HttpClient.class); - - var service = new HttpRequestExecutorService(getTestName(), httpClient, threadPool, null); - - doAnswer(invocation -> { - service.shutdown(); - throw new IllegalArgumentException("failed"); - }).when(httpClient).send(any(), any(), any()); - - PlainActionFuture listener = new PlainActionFuture<>(); - - var request = createHttpPost(0, "a", "b"); - service.send(request, null, listener); - service.start(); - - var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); - assertThat( - thrownException.getMessage(), - is(format("Failed to send request from inference entity id [%s]", request.inferenceEntityId())) - ); - assertThat(thrownException.getCause(), instanceOf(IllegalArgumentException.class)); - assertTrue(service.isTerminated()); - } - - public void testShutdown_AllowsMultipleCalls() { - var service = new HttpRequestExecutorService(getTestName(), mock(HttpClient.class), threadPool, null); - - service.shutdown(); - service.shutdown(); - service.shutdownNow(); - service.start(); - - assertTrue(service.isTerminated()); - assertTrue(service.isShutdown()); - } - - public void testSend_CallsOnFailure_WhenRequestTimesOut() { - var service = new HttpRequestExecutorService("test_service", mock(HttpClient.class), threadPool, null); - - var listener = new PlainActionFuture(); - service.send(HttpRequestTests.createMock("inferenceEntityId"), TimeValue.timeValueNanos(1), listener); - - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); - - assertThat( - thrownException.getMessage(), - is(format("Request timed out waiting to be executed after [%s]", TimeValue.timeValueNanos(1))) - ); - } - - public void testSend_NotifiesTasksOfShutdown() { - var service = new HttpRequestExecutorService("test_service", mock(HttpClient.class), threadPool, null); - - var listener = new PlainActionFuture(); - service.send(HttpRequestTests.createMock("inferenceEntityId"), null, listener); - service.shutdown(); - service.start(); - - var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); - - assertThat( - thrownException.getMessage(), - is("Failed to send request, queue service [test_service] has shutdown prior to executing request") - ); - assertTrue(thrownException.isExecutorShutdown()); - assertTrue(service.isTerminated()); - } - - public void testQueueTake_Throwing_DoesNotCauseServiceToTerminate() throws InterruptedException { - @SuppressWarnings("unchecked") - BlockingQueue queue = mock(LinkedBlockingQueue.class); - when(queue.take()).thenThrow(new ElasticsearchException("failed")).thenReturn(new ShutdownTask()); - - var service = new HttpRequestExecutorService("test_service", mock(HttpClient.class), threadPool, queue, null); - - service.start(); - - assertTrue(service.isTerminated()); - verify(queue, times(2)).take(); - } - - public void testQueueTake_ThrowingInterruptedException_TerminatesService() throws Exception { - @SuppressWarnings("unchecked") - BlockingQueue queue = mock(LinkedBlockingQueue.class); - when(queue.take()).thenThrow(new InterruptedException("failed")); - - var service = new HttpRequestExecutorService("test_service", mock(HttpClient.class), threadPool, queue, null); - - Future executorTermination = threadPool.generic().submit(() -> { - try { - service.start(); - } catch (Exception e) { - fail(Strings.format("Failed to shutdown executor: %s", e)); - } - }); - - executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS); - - assertTrue(service.isTerminated()); - verify(queue, times(1)).take(); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettingsTests.java new file mode 100644 index 0000000000000..c0c0bdd49f617 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettingsTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; + +import static org.elasticsearch.xpack.inference.Utils.mockClusterService; + +public class RequestExecutorServiceSettingsTests { + public static RequestExecutorServiceSettings createRequestExecutorServiceSettingsEmpty() { + return createRequestExecutorServiceSettings(Settings.EMPTY); + } + + public static RequestExecutorServiceSettings createRequestExecutorServiceSettings(@Nullable Integer queueCapacity) { + var settingsBuilder = Settings.builder(); + + if (queueCapacity != null) { + settingsBuilder.put(RequestExecutorServiceSettings.TASK_QUEUE_CAPACITY_SETTING.getKey(), queueCapacity); + } + + return createRequestExecutorServiceSettings(settingsBuilder.build()); + } + + public static RequestExecutorServiceSettings createRequestExecutorServiceSettings(Settings settings) { + return new RequestExecutorServiceSettings(settings, mockClusterService(settings)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java new file mode 100644 index 0000000000000..a4282bbef058d --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java @@ -0,0 +1,426 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.HttpClient; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.HttpRequestTests; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.common.AdjustableCapacityBlockingQueueTests.mockQueueCreator; +import static org.elasticsearch.xpack.inference.external.http.HttpClientTests.createHttpPost; +import static org.elasticsearch.xpack.inference.external.http.sender.RequestExecutorServiceSettingsTests.createRequestExecutorServiceSettings; +import static org.elasticsearch.xpack.inference.external.http.sender.RequestExecutorServiceSettingsTests.createRequestExecutorServiceSettingsEmpty; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RequestExecutorServiceTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private ThreadPool threadPool; + + @Before + public void init() { + threadPool = createThreadPool(inferenceUtilityPool()); + } + + @After + public void shutdown() { + terminate(threadPool); + } + + public void testQueueSize_IsEmpty() { + var service = createRequestExecutorServiceWithMocks(); + + assertThat(service.queueSize(), is(0)); + } + + public void testQueueSize_IsOne() { + var service = createRequestExecutorServiceWithMocks(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); + + assertThat(service.queueSize(), is(1)); + } + + public void testIsTerminated_IsFalse() { + var service = createRequestExecutorServiceWithMocks(); + + assertFalse(service.isTerminated()); + } + + public void testIsTerminated_IsTrue() throws InterruptedException { + var latch = new CountDownLatch(1); + var service = createRequestExecutorService(null, latch); + + service.shutdown(); + service.start(); + latch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); + + assertTrue(service.isTerminated()); + } + + public void testIsTerminated_AfterStopFromSeparateThread() throws Exception { + var waitToShutdown = new CountDownLatch(1); + + var mockHttpClient = mock(HttpClient.class); + doAnswer(invocation -> { + waitToShutdown.countDown(); + return Void.TYPE; + }).when(mockHttpClient).send(any(), any(), any()); + + var service = createRequestExecutorService(mockHttpClient, null); + + Future executorTermination = submitShutdownRequest(waitToShutdown, service); + + PlainActionFuture listener = new PlainActionFuture<>(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, listener); + + service.start(); + + try { + executorTermination.get(1, TimeUnit.SECONDS); + } catch (Exception e) { + fail(Strings.format("Executor finished before it was signaled to shutdown: %s", e)); + } + + assertTrue(service.isShutdown()); + assertTrue(service.isTerminated()); + } + + public void testSend_AfterShutdown_Throws() { + var service = createRequestExecutorServiceWithMocks(); + + service.shutdown(); + + var listener = new PlainActionFuture(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, listener); + + var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); + + assertThat( + thrownException.getMessage(), + is("Failed to enqueue task because the http executor service [test_service] has already shutdown") + ); + assertTrue(thrownException.isExecutorShutdown()); + } + + public void testSend_Throws_WhenQueueIsFull() { + var service = new RequestExecutorService( + "test_service", + mock(HttpClient.class), + threadPool, + null, + RequestExecutorServiceSettingsTests.createRequestExecutorServiceSettings(1) + ); + + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); + var listener = new PlainActionFuture(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, listener); + + var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); + + assertThat( + thrownException.getMessage(), + is("Failed to execute task because the http executor service [test_service] queue is full") + ); + assertFalse(thrownException.isExecutorShutdown()); + } + + public void testTaskThrowsError_CallsOnFailure() throws Exception { + var httpClient = mock(HttpClient.class); + + var service = createRequestExecutorService(httpClient, null); + + doAnswer(invocation -> { + service.shutdown(); + throw new IllegalArgumentException("failed"); + }).when(httpClient).send(any(), any(), any()); + + PlainActionFuture listener = new PlainActionFuture<>(); + + var request = createHttpPost(0, "a", "b"); + service.execute(request, null, listener); + service.start(); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + assertThat( + thrownException.getMessage(), + is(format("Failed to send request from inference entity id [%s]", request.inferenceEntityId())) + ); + assertThat(thrownException.getCause(), instanceOf(IllegalArgumentException.class)); + assertTrue(service.isTerminated()); + } + + public void testShutdown_AllowsMultipleCalls() { + var service = createRequestExecutorServiceWithMocks(); + + service.shutdown(); + service.shutdown(); + service.start(); + + assertTrue(service.isTerminated()); + assertTrue(service.isShutdown()); + } + + public void testSend_CallsOnFailure_WhenRequestTimesOut() { + var service = createRequestExecutorServiceWithMocks(); + + var listener = new PlainActionFuture(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), TimeValue.timeValueNanos(1), listener); + + var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + + assertThat( + thrownException.getMessage(), + is(format("Request timed out waiting to be executed after [%s]", TimeValue.timeValueNanos(1))) + ); + } + + public void testSend_NotifiesTasksOfShutdown() { + var service = createRequestExecutorServiceWithMocks(); + + var listener = new PlainActionFuture(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, listener); + service.shutdown(); + service.start(); + + var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); + + assertThat( + thrownException.getMessage(), + is("Failed to send request, queue service [test_service] has shutdown prior to executing request") + ); + assertTrue(thrownException.isExecutorShutdown()); + assertTrue(service.isTerminated()); + } + + public void testQueueTake_DoesNotCauseServiceToTerminate_WhenItThrows() throws InterruptedException { + @SuppressWarnings("unchecked") + BlockingQueue queue = mock(LinkedBlockingQueue.class); + + var service = new RequestExecutorService( + getTestName(), + mock(HttpClient.class), + threadPool, + mockQueueCreator(queue), + null, + createRequestExecutorServiceSettingsEmpty() + ); + + when(queue.take()).thenThrow(new ElasticsearchException("failed")).thenAnswer(invocation -> { + service.shutdown(); + return null; + }); + service.start(); + + assertTrue(service.isTerminated()); + verify(queue, times(2)).take(); + } + + public void testQueueTake_ThrowingInterruptedException_TerminatesService() throws Exception { + @SuppressWarnings("unchecked") + BlockingQueue queue = mock(LinkedBlockingQueue.class); + when(queue.take()).thenThrow(new InterruptedException("failed")); + + var service = new RequestExecutorService( + getTestName(), + mock(HttpClient.class), + threadPool, + mockQueueCreator(queue), + null, + createRequestExecutorServiceSettingsEmpty() + ); + + Future executorTermination = threadPool.generic().submit(() -> { + try { + service.start(); + } catch (Exception e) { + fail(Strings.format("Failed to shutdown executor: %s", e)); + } + }); + + executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS); + + assertTrue(service.isTerminated()); + verify(queue, times(1)).take(); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105155") + public void testChangingCapacity_SetsCapacityToTwo() throws ExecutionException, InterruptedException, TimeoutException, IOException { + var waitToShutdown = new CountDownLatch(1); + var httpClient = mock(HttpClient.class); + + var settings = createRequestExecutorServiceSettings(1); + var service = new RequestExecutorService("test_service", httpClient, threadPool, null, settings); + + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); + assertThat(service.queueSize(), is(1)); + + PlainActionFuture listener = new PlainActionFuture<>(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, listener); + + var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); + assertThat( + thrownException.getMessage(), + is("Failed to execute task because the http executor service [test_service] queue is full") + ); + + settings.setQueueCapacity(2); + + // There is a request already queued, and its execution path will initiate shutting down the service + doAnswer(invocation -> { + waitToShutdown.countDown(); + return Void.TYPE; + }).when(httpClient).send(any(), any(), any()); + + Future executorTermination = submitShutdownRequest(waitToShutdown, service); + + service.start(); + + executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS); + assertTrue(service.isTerminated()); + assertThat(service.remainingQueueCapacity(), is(2)); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105155") + public void testChangingCapacity_DoesNotRejectsOverflowTasks_BecauseOfQueueFull() throws IOException, ExecutionException, + InterruptedException, TimeoutException { + var waitToShutdown = new CountDownLatch(1); + var httpClient = mock(HttpClient.class); + + var settings = createRequestExecutorServiceSettings(3); + var service = new RequestExecutorService("test_service", httpClient, threadPool, null, settings); + + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); + + PlainActionFuture listener = new PlainActionFuture<>(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, listener); + assertThat(service.queueSize(), is(3)); + + settings.setQueueCapacity(1); + + // There is a request already queued, and its execution path will initiate shutting down the service + doAnswer(invocation -> { + waitToShutdown.countDown(); + return Void.TYPE; + }).when(httpClient).send(any(), any(), any()); + + Future executorTermination = submitShutdownRequest(waitToShutdown, service); + + service.start(); + + executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS); + assertTrue(service.isTerminated()); + assertThat(service.remainingQueueCapacity(), is(1)); + assertThat(service.queueSize(), is(0)); + + var thrownException = expectThrows( + EsRejectedExecutionException.class, + () -> listener.actionGet(TIMEOUT.getSeconds(), TimeUnit.SECONDS) + ); + assertThat( + thrownException.getMessage(), + is("Failed to send request, queue service [test_service] has shutdown prior to executing request") + ); + assertTrue(thrownException.isExecutorShutdown()); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105155") + public void testChangingCapacity_ToZero_SetsQueueCapacityToUnbounded() throws IOException, ExecutionException, InterruptedException, + TimeoutException { + var waitToShutdown = new CountDownLatch(1); + var httpClient = mock(HttpClient.class); + + var settings = createRequestExecutorServiceSettings(1); + var service = new RequestExecutorService("test_service", httpClient, threadPool, null, settings); + + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, new PlainActionFuture<>()); + assertThat(service.queueSize(), is(1)); + + PlainActionFuture listener = new PlainActionFuture<>(); + service.execute(HttpRequestTests.createMock("inferenceEntityId"), null, listener); + + var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT)); + assertThat( + thrownException.getMessage(), + is("Failed to execute task because the http executor service [test_service] queue is full") + ); + + settings.setQueueCapacity(0); + + // There is a request already queued, and its execution path will initiate shutting down the service + doAnswer(invocation -> { + waitToShutdown.countDown(); + return Void.TYPE; + }).when(httpClient).send(any(), any(), any()); + + Future executorTermination = submitShutdownRequest(waitToShutdown, service); + + service.start(); + + executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS); + assertTrue(service.isTerminated()); + assertThat(service.remainingQueueCapacity(), is(Integer.MAX_VALUE)); + } + + private Future submitShutdownRequest(CountDownLatch waitToShutdown, RequestExecutorService service) { + return threadPool.generic().submit(() -> { + try { + // wait for a task to be added to be executed before beginning shutdown + waitToShutdown.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); + service.shutdown(); + service.awaitTermination(TIMEOUT.getSeconds(), TimeUnit.SECONDS); + } catch (Exception e) { + fail(Strings.format("Failed to shutdown executor: %s", e)); + } + }); + } + + private RequestExecutorService createRequestExecutorServiceWithMocks() { + return createRequestExecutorService(null, null); + } + + private RequestExecutorService createRequestExecutorService(@Nullable HttpClient httpClient, @Nullable CountDownLatch startupLatch) { + var httpClientToUse = httpClient == null ? mock(HttpClient.class) : httpClient; + return new RequestExecutorService( + "test_service", + httpClientToUse, + threadPool, + startupLatch, + createRequestExecutorServiceSettingsEmpty() + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryImplTests.java similarity index 92% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryImplTests.java index 2417148c84ac2..fd6a203450c12 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryImplTests.java @@ -45,7 +45,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class ModelRegistryTests extends ESTestCase { +public class ModelRegistryImplTests extends ESTestCase { private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); @@ -65,9 +65,9 @@ public void testGetUnparsedModelMap_ThrowsResourceNotFound_WhenNoHitsReturned() var client = mockClient(); mockClientExecuteSearch(client, mockSearchResponse(SearchHits.EMPTY)); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); ResourceNotFoundException exception = expectThrows(ResourceNotFoundException.class, () -> listener.actionGet(TIMEOUT)); @@ -79,9 +79,9 @@ public void testGetUnparsedModelMap_ThrowsIllegalArgumentException_WhenInvalidIn var unknownIndexHit = SearchHit.createFromMap(Map.of("_index", "unknown_index")); mockClientExecuteSearch(client, mockSearchResponse(new SearchHit[] { unknownIndexHit })); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> listener.actionGet(TIMEOUT)); @@ -96,9 +96,9 @@ public void testGetUnparsedModelMap_ThrowsIllegalStateException_WhenUnableToFind var inferenceSecretsHit = SearchHit.createFromMap(Map.of("_index", ".secrets-inference")); mockClientExecuteSearch(client, mockSearchResponse(new SearchHit[] { inferenceSecretsHit })); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT)); @@ -113,9 +113,9 @@ public void testGetUnparsedModelMap_ThrowsIllegalStateException_WhenUnableToFind var inferenceHit = SearchHit.createFromMap(Map.of("_index", ".inference")); mockClientExecuteSearch(client, mockSearchResponse(new SearchHit[] { inferenceHit })); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT)); @@ -147,9 +147,9 @@ public void testGetModelWithSecrets() { mockClientExecuteSearch(client, mockSearchResponse(new SearchHit[] { inferenceHit, inferenceSecretsHit })); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); var modelConfig = listener.actionGet(TIMEOUT); @@ -176,9 +176,9 @@ public void testGetModelNoSecrets() { mockClientExecuteSearch(client, mockSearchResponse(new SearchHit[] { inferenceHit })); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModel("1", listener); registry.getModel("1", listener); @@ -201,7 +201,7 @@ public void testStoreModel_ReturnsTrue_WhenNoFailuresOccur() { mockClientExecuteBulk(client, bulkResponse); var model = TestModel.createRandomInstance(); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); var listener = new PlainActionFuture(); registry.storeModel(model, listener); @@ -218,7 +218,7 @@ public void testStoreModel_ThrowsException_WhenBulkResponseIsEmpty() { mockClientExecuteBulk(client, bulkResponse); var model = TestModel.createRandomInstance(); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); var listener = new PlainActionFuture(); registry.storeModel(model, listener); @@ -249,7 +249,7 @@ public void testStoreModel_ThrowsResourceAlreadyExistsException_WhenFailureIsAVe mockClientExecuteBulk(client, bulkResponse); var model = TestModel.createRandomInstance(); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); var listener = new PlainActionFuture(); registry.storeModel(model, listener); @@ -275,7 +275,7 @@ public void testStoreModel_ThrowsException_WhenFailureIsNotAVersionConflict() { mockClientExecuteBulk(client, bulkResponse); var model = TestModel.createRandomInstance(); - var registry = new ModelRegistry(client); + var registry = new ModelRegistryImpl(client); var listener = new PlainActionFuture(); registry.storeModel(model, listener); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedSparseEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedSparseEmbeddingResultsTests.java new file mode 100644 index 0000000000000..ea863ea33bd39 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedSparseEmbeddingResultsTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.results; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; + +import java.io.IOException; +import java.util.ArrayList; + +public class ChunkedSparseEmbeddingResultsTests extends AbstractWireSerializingTestCase { + + public static ChunkedSparseEmbeddingResults createRandomResults() { + var chunks = new ArrayList(); + int numChunks = randomIntBetween(1, 5); + + for (int i = 0; i < numChunks; i++) { + var tokenWeights = new ArrayList(); + int numTokens = randomIntBetween(1, 8); + for (int j = 0; j < numTokens; j++) { + tokenWeights.add(new TextExpansionResults.WeightedToken(Integer.toString(j), (float) randomDoubleBetween(0.0, 5.0, false))); + } + chunks.add(new ChunkedTextExpansionResults.ChunkedResult(randomAlphaOfLength(6), tokenWeights)); + } + + return new ChunkedSparseEmbeddingResults(chunks); + } + + @Override + protected Writeable.Reader instanceReader() { + return ChunkedSparseEmbeddingResults::new; + } + + @Override + protected ChunkedSparseEmbeddingResults createTestInstance() { + return createRandomResults(); + } + + @Override + protected ChunkedSparseEmbeddingResults mutateInstance(ChunkedSparseEmbeddingResults instance) throws IOException { + return null; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingResultsTests.java new file mode 100644 index 0000000000000..8a5d41e0e3c1c --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingResultsTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.results; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults; + +import java.io.IOException; +import java.util.ArrayList; + +public class ChunkedTextEmbeddingResultsTests extends AbstractWireSerializingTestCase { + + public static ChunkedTextEmbeddingResults createRandomResults() { + var chunks = new ArrayList(); + int columns = randomIntBetween(5, 10); + int numChunks = randomIntBetween(1, 5); + + for (int i = 0; i < numChunks; i++) { + double[] arr = new double[columns]; + for (int j = 0; j < columns; j++) { + arr[j] = randomDouble(); + } + chunks.add( + new org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults.EmbeddingChunk( + randomAlphaOfLength(6), + arr + ) + ); + } + + return new ChunkedTextEmbeddingResults(chunks); + } + + @Override + protected Writeable.Reader instanceReader() { + return ChunkedTextEmbeddingResults::new; + } + + @Override + protected ChunkedTextEmbeddingResults createTestInstance() { + return createRandomResults(); + } + + @Override + protected ChunkedTextEmbeddingResults mutateInstance(ChunkedTextEmbeddingResults instance) throws IOException { + return null; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java index 8b596aa5cf0c8..873582ee353c7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -112,6 +114,18 @@ protected void doInfer( } + @Override + protected void doChunkedInfer( + Model model, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + ActionListener listener + ) { + + } + @Override public String name() { return "test service"; diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index b6215b4efe5ba..819f41781a307 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.License; @@ -44,6 +45,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -85,7 +87,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestPutPipelineAction(), new RestGetPipelineAction(), new RestDeletePipelineAction()); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index cc819c353f69c..cee397d906149 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -311,7 +311,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (fieldType().value == null) { ConstantKeywordFieldType newFieldType = new ConstantKeywordFieldType(fieldType().name(), value, fieldType().meta()); Mapper update = new ConstantKeywordFieldMapper(simpleName(), newFieldType); - context.addDynamicMapper(update); + boolean dynamicMapperAdded = context.addDynamicMapper(update); + // the mapper is already part of the mapping, we're just updating it with the new value + assert dynamicMapperAdded; } else if (Objects.equals(fieldType().value, value) == false) { throw new IllegalArgumentException( "[constant_keyword] field [" diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordMapperPlugin.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordMapperPlugin.java index 62fb10be05f9d..43610ecede072 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordMapperPlugin.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordMapperPlugin.java @@ -11,6 +11,7 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.bucket.countedterms.CountedTermsAggregationBuilder; import java.util.ArrayList; import java.util.Collections; diff --git a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilderTests.java b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilderTests.java index ba266e82fecc8..00740d8a0bd20 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilderTests.java +++ b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilderTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; +import org.elasticsearch.search.aggregations.bucket.countedterms.CountedTermsAggregationBuilder; import java.util.Collection; import java.util.Collections; diff --git a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorTests.java b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorTests.java index 02d629c7604ac..ef11c7dd3e9d9 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorTests.java +++ b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregatorTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.TestDocumentParserContext; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.countedterms.CountedTermsAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.xcontent.XContentParser; diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index e8fdf7e0205da..c468d7bcd6718 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -195,6 +195,9 @@ Number parsedNullValue() { @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { + if (context.parentObjectContainsDimensions()) { + dimension.setValue(true); + } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( context.buildFullName(name), indexed.getValue(), @@ -637,7 +640,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } if (dimension && numericValue != null) { - context.getDimensions().addUnsignedLong(fieldType().name(), numericValue); + context.getDimensions().addUnsignedLong(fieldType().name(), numericValue).validate(context.indexSettings()); } List fields = new ArrayList<>(); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 95fe8f0a530ba..fc783ef92a112 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -19,9 +19,10 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MapperTestCase; +import org.elasticsearch.index.mapper.NumberTypeOutOfRangeSpec; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.TimeSeriesParams; +import org.elasticsearch.index.mapper.WholeNumberFieldMapperTests; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -29,6 +30,7 @@ import java.io.IOException; import java.math.BigInteger; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Function; @@ -38,7 +40,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesPattern; -public class UnsignedLongFieldMapperTests extends MapperTestCase { +public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests { @Override protected Collection getPlugins() { @@ -161,6 +163,9 @@ public void testNullValue() throws IOException { } } + @Override + public void testCoerce() {} // coerce is unimplemented + @Override protected boolean supportsIgnoreMalformed() { return true; @@ -367,6 +372,36 @@ protected IngestScriptSupport ingestScriptSupport() { } @Override + protected List outOfRangeSpecs() { + return Collections.emptyList(); // unimplemented + } + + @Override + public void testIgnoreMalformedWithObject() {} // unimplemented + + @Override + public void testAllowMultipleValuesField() {} // unimplemented + + @Override + public void testScriptableTypes() {} // unimplemented + + @Override + protected Number missingValue() { + return 123L; + } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomLong(); + } + if (randomBoolean()) { + return randomDouble(); + } + assumeFalse("https://github.com/elastic/elasticsearch/issues/70585", true); + return randomDoubleBetween(0L, Long.MAX_VALUE, true); + } + protected Function loadBlockExpected() { return v -> { // Numbers are in the block as a long but the test needs to compare them to their BigInteger value parsed from xcontent. diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml index 150c90faf175a..5d4a971c6fd1b 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml +++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml @@ -64,10 +64,10 @@ fetch the _tsid: sort: [ _tsid ] - match: {hits.total.value: 2} - - match: {hits.hits.0.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} + - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} - match: {hits.hits.0.fields.metricset: [aa]} - match: {hits.hits.0.fields.ul: [9223372036854775807]} - - match: {hits.hits.1.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} + - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} - match: {hits.hits.1.fields.metricset: [aa]} - match: {hits.hits.1.fields.ul: [9223372036854775808]} @@ -119,14 +119,14 @@ aggregate the _tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {metricset: aa, ul: 9223372036854775807}} - - match: {aggregations.tsids.buckets.0.doc_count: 3} - - match: {aggregations.tsids.buckets.1.key: {metricset: aa, ul: 9223372036854775808}} + - match: {aggregations.tsids.buckets.0.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"} + - match: {aggregations.tsids.buckets.0.doc_count: 1} + - match: {aggregations.tsids.buckets.1.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"} - match: {aggregations.tsids.buckets.1.doc_count: 3} - - match: {aggregations.tsids.buckets.2.key: {metricset: aa, ul: 18446744073709551614}} + - match: {aggregations.tsids.buckets.2.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"} - match: {aggregations.tsids.buckets.2.doc_count: 1} - - match: {aggregations.tsids.buckets.3.key: {metricset: aa, ul: 18446744073709551615}} - - match: {aggregations.tsids.buckets.3.doc_count: 1 } + - match: {aggregations.tsids.buckets.3.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"} + - match: {aggregations.tsids.buckets.3.doc_count: 3} --- @@ -147,14 +147,14 @@ sort by tsid: - match: {hits.total.value: 8 } - - match: {hits.hits.0.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} - - match: {hits.hits.1.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} - - match: {hits.hits.2.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} - - match: {hits.hits.3.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} - - match: {hits.hits.4.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} - - match: {hits.hits.5.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} - - match: {hits.hits.6.fields._tsid: [{metricset: aa, ul: 18446744073709551614}]} - - match: {hits.hits.7.fields._tsid: [{metricset: aa, ul: 18446744073709551615}]} + - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"]} + - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.2.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.3.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.4.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"]} + - match: {hits.hits.5.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.6.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.7.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} --- composite aggregation on tsid: @@ -180,18 +180,13 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid.ul: 9223372036854775807 } - - match: { aggregations.tsids.buckets.0.key.tsid.metricset: "aa" } - - match: { aggregations.tsids.buckets.0.doc_count: 3 } - - match: { aggregations.tsids.buckets.1.key.tsid.ul: 9223372036854775808 } - - match: { aggregations.tsids.buckets.1.key.tsid.metricset: "aa" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ" } + - match: { aggregations.tsids.buckets.0.doc_count: 1 } + - match: { aggregations.tsids.buckets.1.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA" } - match: { aggregations.tsids.buckets.1.doc_count: 3 } - - match: { aggregations.tsids.buckets.2.key.tsid.ul: 18446744073709551614 } - - match: { aggregations.tsids.buckets.2.key.tsid.metricset: "aa" } + - match: { aggregations.tsids.buckets.2.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM" } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - match: { aggregations.tsids.buckets.3.key.tsid.ul: 18446744073709551615 } - - match: { aggregations.tsids.buckets.3.key.tsid.metricset: "aa" } - - match: { aggregations.tsids.buckets.3.doc_count: 1 } + - match: { aggregations.tsids.buckets.3.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } + - match: { aggregations.tsids.buckets.3.doc_count: 3 } - - match: { aggregations.tsids.after_key.tsid.ul: 18446744073709551615 } - - match: { aggregations.tsids.after_key.tsid.metricset: "aa" } + - match: { aggregations.tsids.after_key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } diff --git a/x-pack/plugin/ml/qa/disabled/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlPluginDisabledIT.java b/x-pack/plugin/ml/qa/disabled/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlPluginDisabledIT.java index 25ea9c5d2b27e..a518e0d496868 100644 --- a/x-pack/plugin/ml/qa/disabled/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlPluginDisabledIT.java +++ b/x-pack/plugin/ml/qa/disabled/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlPluginDisabledIT.java @@ -10,14 +10,19 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; public class MlPluginDisabledIT extends ESRestTestCase { @@ -71,7 +76,19 @@ public void testActionsFail() throws Exception { public void testMlFeatureReset() throws IOException { Request request = new Request("POST", "/_features/_reset"); - Response response = client().performRequest(request); - assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertOK(client().performRequest(request)); + } + + @SuppressWarnings("unchecked") + public void testAllNodesHaveMlConfigVersionAttribute() throws IOException { + Request request = new Request("GET", "/_nodes"); + Response response = assertOK(client().performRequest(request)); + var nodesMap = (Map) entityAsMap(response).get("nodes"); + assertThat(nodesMap, is(aMapWithSize(greaterThanOrEqualTo(1)))); + for (var nodeObj : nodesMap.values()) { + var nodeMap = (Map) nodeObj; + // We do not expect any specific version. The only important assertion is that the attribute exists. + assertThat(XContentMapValues.extractValue(nodeMap, "attributes", "ml.config_version"), is(notNullValue())); + } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 152d8fde8c86c..6916a04084285 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.mapper.Mapper; @@ -466,6 +467,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -827,13 +829,19 @@ public Settings additionalSettings() { String allocatedProcessorsAttrName = "node.attr." + ALLOCATED_PROCESSORS_NODE_ATTR; String mlConfigVersionAttrName = "node.attr." + ML_CONFIG_VERSION_NODE_ATTR; - if (enabled == false) { - disallowMlNodeAttributes(maxOpenJobsPerNodeNodeAttrName, machineMemoryAttrName, jvmSizeAttrName, mlConfigVersionAttrName); - return Settings.EMPTY; - } - Settings.Builder additionalSettings = Settings.builder(); - if (DiscoveryNode.hasRole(settings, DiscoveryNodeRole.ML_ROLE)) { + + // The ML config version is needed for two related reasons even if ML is currently disabled on the node: + // 1. If ML is in use then decisions about minimum node versions need to include this node, and not + // having it available can cause exceptions during cluster state processing + // 2. It could be argued that reason 1 could be fixed by completely ignoring the node, however, + // then there would be a risk that ML is later enabled on an old node that was ignored, and + // some new ML feature that's been used is then incompatible with it + // The only safe approach is to consider which ML code _all_ nodes in the cluster are running, regardless + // of whether they currently have ML enabled. + addMlNodeAttribute(additionalSettings, mlConfigVersionAttrName, MlConfigVersion.CURRENT.toString()); + + if (enabled && DiscoveryNode.hasRole(settings, DiscoveryNodeRole.ML_ROLE)) { addMlNodeAttribute( additionalSettings, machineMemoryAttrName, @@ -857,7 +865,6 @@ public Settings additionalSettings() { allocatedProcessorsAttrName ); } - addMlNodeAttribute(additionalSettings, mlConfigVersionAttrName, MlConfigVersion.CURRENT.toString()); return additionalSettings.build(); } @@ -1367,7 +1374,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (false == enabled) { return List.of(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java index 65d630ebf1d6e..2760c2990fadf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java @@ -109,6 +109,7 @@ protected void taskOperation( request.getInferenceTimeout(), request.getPrefixType(), actionTask, + request.isChunkResults(), orderedListener(count, results, slot++, nlpInputs.size(), listener) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractor.java index 991916333f4cf..0ea1914d3e14b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractor.java @@ -6,8 +6,6 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor; -import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.xpack.core.ml.datafeed.SearchInterval; import java.io.IOException; @@ -18,6 +16,14 @@ public interface DataExtractor { record Result(SearchInterval searchInterval, Optional data) {} + record DataSummary(Long earliestTime, Long latestTime, long totalHits) { + public boolean hasData() { + return earliestTime != null; + } + } + + DataSummary getSummary(); + /** * @return {@code true} if the search has not finished yet, or {@code false} otherwise */ @@ -50,22 +56,4 @@ record Result(SearchInterval searchInterval, Optional data) {} * @return the end time to which this extractor will search */ long getEndTime(); - - /** - * Check whether the search skipped CCS clusters. - * @throws ResourceNotFoundException if any CCS clusters were skipped, as this could - * cause anomalies to be spuriously detected. - * @param searchResponse The search response to check for skipped CCS clusters. - */ - default void checkForSkippedClusters(SearchResponse searchResponse) { - SearchResponse.Clusters clusterResponse = searchResponse.getClusters(); - if (clusterResponse != null && clusterResponse.getClusterStateCount(SearchResponse.Cluster.Status.SKIPPED) > 0) { - throw new ResourceNotFoundException( - "[{}] remote clusters out of [{}] were skipped when performing datafeed search", - clusterResponse.getClusterStateCount(SearchResponse.Cluster.Status.SKIPPED), - clusterResponse.getTotal() - ); - } - } - } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java index 3175891aa4d6e..be2c8dd871a9b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.ml.datafeed.extractor.scroll.ScrollDataExtractorFactory; public interface DataExtractorFactory { + DataExtractor newExtractor(long start, long end); /** @@ -61,7 +62,7 @@ static void create( ActionListener factoryHandler = ActionListener.wrap( factory -> listener.onResponse( datafeed.getChunkingConfig().isEnabled() - ? new ChunkedDataExtractorFactory(client, datafeed, extraFilters, job, xContentRegistry, factory, timingStatsReporter) + ? new ChunkedDataExtractorFactory(datafeed, job, xContentRegistry, factory) : factory ), listener::onFailure diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorQueryContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorQueryContext.java new file mode 100644 index 0000000000000..8ba901f82d351 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorQueryContext.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.datafeed.extractor; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.index.query.QueryBuilder; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class DataExtractorQueryContext { + + public final String[] indices; + public final QueryBuilder query; + public final String timeField; + public final long start; + public final long end; + public final Map headers; + public final IndicesOptions indicesOptions; + public final Map runtimeMappings; + + public DataExtractorQueryContext( + List indices, + QueryBuilder query, + String timeField, + long start, + long end, + Map headers, + IndicesOptions indicesOptions, + Map runtimeMappings + ) { + this.indices = indices.toArray(new String[0]); + this.query = Objects.requireNonNull(query); + this.timeField = timeField; + this.start = start; + this.end = end; + this.headers = headers; + this.indicesOptions = Objects.requireNonNull(indicesOptions); + this.runtimeMappings = Objects.requireNonNull(runtimeMappings); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java index 0f6ae6f90fb52..f0e03a1e94973 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorUtils.java @@ -7,9 +7,18 @@ package org.elasticsearch.xpack.ml.datafeed.extractor; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.metrics.Max; +import org.elasticsearch.search.aggregations.metrics.Min; +import org.elasticsearch.search.builder.SearchSourceBuilder; /** * Utility methods for various DataExtractor implementations. @@ -17,12 +26,70 @@ public final class DataExtractorUtils { private static final String EPOCH_MILLIS = "epoch_millis"; + private static final String EARLIEST_TIME = "earliest_time"; + private static final String LATEST_TIME = "latest_time"; + + private DataExtractorUtils() {} /** * Combines a user query with a time range query. */ - public static QueryBuilder wrapInTimeRangeQuery(QueryBuilder userQuery, String timeField, long start, long end) { + public static QueryBuilder wrapInTimeRangeQuery(QueryBuilder query, String timeField, long start, long end) { QueryBuilder timeQuery = new RangeQueryBuilder(timeField).gte(start).lt(end).format(EPOCH_MILLIS); - return new BoolQueryBuilder().filter(userQuery).filter(timeQuery); + return new BoolQueryBuilder().filter(query).filter(timeQuery); + } + + public static SearchRequestBuilder getSearchRequestBuilderForSummary(Client client, DataExtractorQueryContext context) { + return new SearchRequestBuilder(client).setIndices(context.indices) + .setIndicesOptions(context.indicesOptions) + .setSource(getSearchSourceBuilderForSummary(context)) + .setAllowPartialSearchResults(false) + .setTrackTotalHits(true); + } + + public static SearchSourceBuilder getSearchSourceBuilderForSummary(DataExtractorQueryContext context) { + return new SearchSourceBuilder().size(0) + .query(DataExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, context.start, context.end)) + .runtimeMappings(context.runtimeMappings) + .aggregation(AggregationBuilders.min(EARLIEST_TIME).field(context.timeField)) + .aggregation(AggregationBuilders.max(LATEST_TIME).field(context.timeField)); + } + + public static DataExtractor.DataSummary getDataSummary(SearchResponse searchResponse) { + InternalAggregations aggregations = searchResponse.getAggregations(); + if (aggregations == null) { + return new DataExtractor.DataSummary(null, null, 0L); + } else { + Long earliestTime = toLongIfFinite((aggregations.get(EARLIEST_TIME)).value()); + Long latestTime = toLongIfFinite((aggregations.get(LATEST_TIME)).value()); + long totalHits = searchResponse.getHits().getTotalHits().value; + return new DataExtractor.DataSummary(earliestTime, latestTime, totalHits); + } + } + + /** + * The min and max aggregations return infinity when there is no data. To ensure consistency + * between the different types of data summary we represent no data by earliest and latest times + * being null. Hence, this method converts infinite values to null. + */ + private static Long toLongIfFinite(double x) { + return Double.isFinite(x) ? (long) x : null; + } + + /** + * Check whether the search skipped CCS clusters. + * @throws ResourceNotFoundException if any CCS clusters were skipped, as this could + * cause anomalies to be spuriously detected. + * @param searchResponse The search response to check for skipped CCS clusters. + */ + public static void checkForSkippedClusters(SearchResponse searchResponse) { + SearchResponse.Clusters clusterResponse = searchResponse.getClusters(); + if (clusterResponse != null && clusterResponse.getClusterStateCount(SearchResponse.Cluster.Status.SKIPPED) > 0) { + throw new ResourceNotFoundException( + "[{}] remote clusters out of [{}] were skipped when performing datafeed search", + clusterResponse.getClusterStateCount(SearchResponse.Cluster.Status.SKIPPED), + clusterResponse.getTotal() + ); + } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java index 4cd5379d8fe3b..26c43e1d098c1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java @@ -34,10 +34,8 @@ /** * Abstract class for aggregated data extractors, e.g. {@link RollupDataExtractor} - * - * @param The request builder type for getting data from ElasticSearch */ -abstract class AbstractAggregationDataExtractor> implements DataExtractor { +abstract class AbstractAggregationDataExtractor implements DataExtractor { private static final Logger LOGGER = LogManager.getLogger(AbstractAggregationDataExtractor.class); @@ -86,7 +84,7 @@ public void destroy() { @Override public long getEndTime() { - return context.end; + return context.queryContext.end; } @Override @@ -95,7 +93,7 @@ public Result next() throws IOException { throw new NoSuchElementException(); } - SearchInterval searchInterval = new SearchInterval(context.start, context.end); + SearchInterval searchInterval = new SearchInterval(context.queryContext.start, context.queryContext.end); if (aggregationToJsonProcessor == null) { InternalAggregations aggs = search(); if (aggs == null) { @@ -121,11 +119,10 @@ public Result next() throws IOException { private InternalAggregations search() { LOGGER.debug("[{}] Executing aggregated search", context.jobId); - T searchRequest = buildSearchRequest(buildBaseSearchSource()); + ActionRequestBuilder searchRequest = buildSearchRequest(buildBaseSearchSource()); assert searchRequest.request().allowPartialSearchResults() == false; SearchResponse searchResponse = executeSearchRequest(searchRequest); try { - checkForSkippedClusters(searchResponse); LOGGER.debug("[{}] Search response was obtained", context.jobId); timingStatsReporter.reportSearchDuration(searchResponse.getTook()); return validateAggs(searchResponse.getAggregations()); @@ -136,37 +133,62 @@ private InternalAggregations search() { private void initAggregationProcessor(InternalAggregations aggs) throws IOException { aggregationToJsonProcessor = new AggregationToJsonProcessor( - context.timeField, + context.queryContext.timeField, context.fields, context.includeDocCount, - context.start, + context.queryContext.start, null ); aggregationToJsonProcessor.process(aggs); } - protected SearchResponse executeSearchRequest(T searchRequestBuilder) { - return ClientHelper.executeWithHeaders(context.headers, ClientHelper.ML_ORIGIN, client, searchRequestBuilder::get); + private SearchResponse executeSearchRequest(ActionRequestBuilder searchRequestBuilder) { + SearchResponse searchResponse = ClientHelper.executeWithHeaders( + context.queryContext.headers, + ClientHelper.ML_ORIGIN, + client, + searchRequestBuilder::get + ); + boolean success = false; + try { + DataExtractorUtils.checkForSkippedClusters(searchResponse); + success = true; + } finally { + if (success == false) { + searchResponse.decRef(); + } + } + return searchResponse; } private SearchSourceBuilder buildBaseSearchSource() { // For derivative aggregations the first bucket will always be null // so query one extra histogram bucket back and hope there is data // in that bucket - long histogramSearchStartTime = Math.max(0, context.start - DatafeedConfigUtils.getHistogramIntervalMillis(context.aggs)); + long histogramSearchStartTime = Math.max( + 0, + context.queryContext.start - DatafeedConfigUtils.getHistogramIntervalMillis(context.aggs) + ); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0) - .query(DataExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, histogramSearchStartTime, context.end)); + .query( + DataExtractorUtils.wrapInTimeRangeQuery( + context.queryContext.query, + context.queryContext.timeField, + histogramSearchStartTime, + context.queryContext.end + ) + ); - if (context.runtimeMappings.isEmpty() == false) { - searchSourceBuilder.runtimeMappings(context.runtimeMappings); + if (context.queryContext.runtimeMappings.isEmpty() == false) { + searchSourceBuilder.runtimeMappings(context.queryContext.runtimeMappings); } context.aggs.getAggregatorFactories().forEach(searchSourceBuilder::aggregation); context.aggs.getPipelineAggregatorFactories().forEach(searchSourceBuilder::aggregation); return searchSourceBuilder; } - protected abstract T buildSearchRequest(SearchSourceBuilder searchRequestBuilder); + protected abstract ActionRequestBuilder buildSearchRequest(SearchSourceBuilder searchRequestBuilder); private static InternalAggregations validateAggs(@Nullable InternalAggregations aggs) { if (aggs == null) { @@ -189,4 +211,18 @@ public AggregationDataExtractorContext getContext() { return context; } + @Override + public DataSummary getSummary() { + ActionRequestBuilder searchRequestBuilder = buildSearchRequest( + DataExtractorUtils.getSearchSourceBuilderForSummary(context.queryContext) + ); + SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder); + try { + LOGGER.debug("[{}] Aggregating Data summary response was obtained", context.jobId); + timingStatsReporter.reportSearchDuration(searchResponse.getTook()); + return DataExtractorUtils.getDataSummary(searchResponse); + } finally { + searchResponse.decRef(); + } + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java index 34ea3a1fad04e..0a41c4387634e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java @@ -17,7 +17,7 @@ * stored and they are then processed in batches. Cancellation is supported between batches. * Note that this class is NOT thread-safe. */ -class AggregationDataExtractor extends AbstractAggregationDataExtractor { +class AggregationDataExtractor extends AbstractAggregationDataExtractor { AggregationDataExtractor( Client client, @@ -30,8 +30,8 @@ class AggregationDataExtractor extends AbstractAggregationDataExtractor fields; - final String[] indices; - final QueryBuilder query; final AggregatorFactories.Builder aggs; - final long start; - final long end; final boolean includeDocCount; - final Map headers; - final IndicesOptions indicesOptions; - final Map runtimeMappings; + final DataExtractorQueryContext queryContext; AggregationDataExtractorContext( String jobId, @@ -44,17 +37,19 @@ class AggregationDataExtractorContext { IndicesOptions indicesOptions, Map runtimeMappings ) { - this.jobId = Objects.requireNonNull(jobId); - this.timeField = Objects.requireNonNull(timeField); + this.jobId = jobId; this.fields = Objects.requireNonNull(fields); - this.indices = indices.toArray(new String[0]); - this.query = Objects.requireNonNull(query); this.aggs = Objects.requireNonNull(aggs); - this.start = start; - this.end = end; this.includeDocCount = includeDocCount; - this.headers = headers; - this.indicesOptions = Objects.requireNonNull(indicesOptions); - this.runtimeMappings = Objects.requireNonNull(runtimeMappings); + this.queryContext = new DataExtractorQueryContext( + indices, + query, + Objects.requireNonNull(timeField), + start, + end, + headers, + indicesOptions, + runtimeMappings + ); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java index 0dfdd9897737e..e4712d051ef1e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java @@ -48,6 +48,9 @@ class CompositeAggregationDataExtractor implements DataExtractor { private static final Logger LOGGER = LogManager.getLogger(CompositeAggregationDataExtractor.class); + private static final String EARLIEST_TIME = "earliest_time"; + private static final String LATEST_TIME = "latest_time"; + private volatile Map afterKey = null; private final CompositeAggregationBuilder compositeAggregationBuilder; private final Client client; @@ -98,7 +101,7 @@ public void destroy() { @Override public long getEndTime() { - return context.end; + return context.queryContext.end; } @Override @@ -107,7 +110,7 @@ public Result next() throws IOException { throw new NoSuchElementException(); } - SearchInterval searchInterval = new SearchInterval(context.start, context.end); + SearchInterval searchInterval = new SearchInterval(context.queryContext.start, context.queryContext.end); InternalAggregations aggs = search(); if (aggs == null) { LOGGER.trace(() -> "[" + context.jobId + "] extraction finished"); @@ -125,13 +128,25 @@ private InternalAggregations search() { // Also, it doesn't make sense to have a derivative when grouping by time AND by some other criteria. LOGGER.trace( - () -> format("[%s] Executing composite aggregated search from [%s] to [%s]", context.jobId, context.start, context.end) + () -> format( + "[%s] Executing composite aggregated search from [%s] to [%s]", + context.jobId, + context.queryContext.start, + context.queryContext.end + ) ); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0) - .query(DataExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, context.start, context.end)); + .query( + DataExtractorUtils.wrapInTimeRangeQuery( + context.queryContext.query, + context.queryContext.timeField, + context.queryContext.start, + context.queryContext.end + ) + ); - if (context.runtimeMappings.isEmpty() == false) { - searchSourceBuilder.runtimeMappings(context.runtimeMappings); + if (context.queryContext.runtimeMappings.isEmpty() == false) { + searchSourceBuilder.runtimeMappings(context.queryContext.runtimeMappings); } if (afterKey != null) { compositeAggregationBuilder.aggregateAfter(afterKey); @@ -156,16 +171,16 @@ private InternalAggregations search() { } } - protected SearchResponse executeSearchRequest(ActionRequestBuilder searchRequestBuilder) { + private SearchResponse executeSearchRequest(ActionRequestBuilder searchRequestBuilder) { SearchResponse searchResponse = ClientHelper.executeWithHeaders( - context.headers, + context.queryContext.headers, ClientHelper.ML_ORIGIN, client, searchRequestBuilder::get ); boolean success = false; try { - checkForSkippedClusters(searchResponse); + DataExtractorUtils.checkForSkippedClusters(searchResponse); success = true; } finally { if (success == false) { @@ -177,10 +192,10 @@ protected SearchResponse executeSearchRequest(ActionRequestBuilder searchRequestBuilder = DataExtractorUtils.getSearchRequestBuilderForSummary( + client, + context.queryContext + ); + SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder); + try { + LOGGER.debug("[{}] Aggregating Data summary response was obtained", context.jobId); + timingStatsReporter.reportSearchDuration(searchResponse.getTook()); + return DataExtractorUtils.getDataSummary(searchResponse); + } finally { + searchResponse.decRef(); + } + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorContext.java index 5fd5b58c5556d..75531e68d9738 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorContext.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorQueryContext; import java.util.List; import java.util.Map; @@ -18,18 +19,11 @@ class CompositeAggregationDataExtractorContext { final String jobId; - final String timeField; final Set fields; - final String[] indices; - final QueryBuilder query; final CompositeAggregationBuilder compositeAggregationBuilder; - final long start; - final long end; final boolean includeDocCount; - final Map headers; - final IndicesOptions indicesOptions; - final Map runtimeMappings; final String compositeAggDateHistogramGroupSourceName; + final DataExtractorQueryContext queryContext; CompositeAggregationDataExtractorContext( String jobId, @@ -47,17 +41,19 @@ class CompositeAggregationDataExtractorContext { Map runtimeMappings ) { this.jobId = Objects.requireNonNull(jobId); - this.timeField = Objects.requireNonNull(timeField); this.fields = Objects.requireNonNull(fields); - this.indices = indices.toArray(new String[0]); - this.query = Objects.requireNonNull(query); this.compositeAggregationBuilder = Objects.requireNonNull(compositeAggregationBuilder); this.compositeAggDateHistogramGroupSourceName = Objects.requireNonNull(compositeAggDateHistogramGroupSourceName); - this.start = start; - this.end = end; this.includeDocCount = includeDocCount; - this.headers = headers; - this.indicesOptions = Objects.requireNonNull(indicesOptions); - this.runtimeMappings = Objects.requireNonNull(runtimeMappings); + this.queryContext = new DataExtractorQueryContext( + indices, + query, + Objects.requireNonNull(timeField), + start, + end, + headers, + indicesOptions, + runtimeMappings + ); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java index 1503e93e5c11b..89a137807959d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java @@ -18,7 +18,7 @@ * stored and they are then processed in batches. Cancellation is supported between batches. * Note that this class is NOT thread-safe. */ -class RollupDataExtractor extends AbstractAggregationDataExtractor { +class RollupDataExtractor extends AbstractAggregationDataExtractor { RollupDataExtractor( Client client, @@ -30,8 +30,8 @@ class RollupDataExtractor extends AbstractAggregationDataExtractor The chunk span can be either specified or not. When not specified, - * a heuristic is employed (see {@link DataSummary#estimateChunk()}) to automatically determine the chunk span. - * The search is set up (see {@link #setUpChunkedSearch()} by querying a data summary for the given time range + * a heuristic is employed (see {@link #setUpChunkedSearch()}) to automatically determine the chunk span. + * The search is set up by querying a data summary for the given time range * that includes the number of total hits and the earliest/latest times. Those are then used to determine the chunk span, * when necessary, and to jump the search forward to the time where the earliest data can be found. * If a search for a chunk returns empty, the set up is performed again for the remaining time. @@ -50,49 +35,30 @@ */ public class ChunkedDataExtractor implements DataExtractor { - interface DataSummary { - long estimateChunk(); - - boolean hasData(); - - long earliestTime(); - - long getDataTimeSpread(); - } - private static final Logger LOGGER = LogManager.getLogger(ChunkedDataExtractor.class); - private static final String EARLIEST_TIME = "earliest_time"; - private static final String LATEST_TIME = "latest_time"; - /** Let us set a minimum chunk span of 1 minute */ private static final long MIN_CHUNK_SPAN = 60000L; - private final Client client; private final DataExtractorFactory dataExtractorFactory; private final ChunkedDataExtractorContext context; - private final DataSummaryFactory dataSummaryFactory; - private final DatafeedTimingStatsReporter timingStatsReporter; private long currentStart; private long currentEnd; private long chunkSpan; private boolean isCancelled; private DataExtractor currentExtractor; - public ChunkedDataExtractor( - Client client, - DataExtractorFactory dataExtractorFactory, - ChunkedDataExtractorContext context, - DatafeedTimingStatsReporter timingStatsReporter - ) { - this.client = Objects.requireNonNull(client); + public ChunkedDataExtractor(DataExtractorFactory dataExtractorFactory, ChunkedDataExtractorContext context) { this.dataExtractorFactory = Objects.requireNonNull(dataExtractorFactory); this.context = Objects.requireNonNull(context); - this.timingStatsReporter = Objects.requireNonNull(timingStatsReporter); - this.currentStart = context.start; - this.currentEnd = context.start; + this.currentStart = context.start(); + this.currentEnd = context.start(); this.isCancelled = false; - this.dataSummaryFactory = new DataSummaryFactory(); + } + + @Override + public DataSummary getSummary() { + return null; } @Override @@ -101,7 +67,7 @@ public boolean hasNext() { if (isCancelled()) { return currentHasNext; } - return currentHasNext || currentEnd < context.end; + return currentHasNext || currentEnd < context.end(); } @Override @@ -119,47 +85,42 @@ public Result next() throws IOException { } private void setUpChunkedSearch() { - DataSummary dataSummary = dataSummaryFactory.buildDataSummary(); + DataSummary dataSummary = dataExtractorFactory.newExtractor(currentStart, context.end()).getSummary(); if (dataSummary.hasData()) { - currentStart = context.timeAligner.alignToFloor(dataSummary.earliestTime()); + currentStart = context.timeAligner().alignToFloor(dataSummary.earliestTime()); currentEnd = currentStart; - chunkSpan = context.chunkSpan == null ? dataSummary.estimateChunk() : context.chunkSpan.getMillis(); - chunkSpan = context.timeAligner.alignToCeil(chunkSpan); - LOGGER.debug( - "[{}] Chunked search configured: kind = {}, dataTimeSpread = {} ms, chunk span = {} ms", - context.jobId, - dataSummary.getClass().getSimpleName(), - dataSummary.getDataTimeSpread(), - chunkSpan - ); - } else { - // search is over - currentEnd = context.end; - LOGGER.debug("[{}] Chunked search configured: no data found", context.jobId); - } - } - protected SearchResponse executeSearchRequest(ActionRequestBuilder searchRequestBuilder) { - SearchResponse searchResponse = ClientHelper.executeWithHeaders( - context.headers, - ClientHelper.ML_ORIGIN, - client, - searchRequestBuilder::get - ); - boolean success = false; - try { - checkForSkippedClusters(searchResponse); - success = true; - } finally { - if (success == false) { - searchResponse.decRef(); + if (context.chunkSpan() != null) { + chunkSpan = context.chunkSpan().getMillis(); + } else if (context.hasAggregations()) { + // This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig} + chunkSpan = DatafeedConfig.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * context.histogramInterval(); + } else { + long timeSpread = dataSummary.latestTime() - dataSummary.earliestTime(); + if (timeSpread <= 0) { + chunkSpan = context.end() - currentEnd; + } else { + // The heuristic here is that we want a time interval where we expect roughly scrollSize documents + // (assuming data are uniformly spread over time). + // We have totalHits documents over dataTimeSpread (latestTime - earliestTime), we want scrollSize documents over chunk. + // Thus, the interval would be (scrollSize * dataTimeSpread) / totalHits. + // However, assuming this as the chunk span may often lead to half-filled pages or empty searches. + // It is beneficial to take a multiple of that. Based on benchmarking, we set this to 10x. + chunkSpan = Math.max(MIN_CHUNK_SPAN, 10 * (context.scrollSize() * timeSpread) / dataSummary.totalHits()); + } } + + chunkSpan = context.timeAligner().alignToCeil(chunkSpan); + LOGGER.debug("[{}] Chunked search configured: chunk span = {} ms", context.jobId(), chunkSpan); + } else { + // search is over + currentEnd = context.end(); + LOGGER.debug("[{}] Chunked search configured: no data found", context.jobId()); } - return searchResponse; } private Result getNextStream() throws IOException { - SearchInterval lastSearchInterval = new SearchInterval(context.start, context.end); + SearchInterval lastSearchInterval = new SearchInterval(context.start(), context.end()); while (hasNext()) { boolean isNewSearch = false; @@ -202,9 +163,9 @@ private Result getNextStream() throws IOException { private void advanceTime() { currentStart = currentEnd; - currentEnd = Math.min(currentStart + chunkSpan, context.end); + currentEnd = Math.min(currentStart + chunkSpan, context.end()); currentExtractor = dataExtractorFactory.newExtractor(currentStart, currentEnd); - LOGGER.trace("[{}] advances time to [{}, {})", context.jobId, currentStart, currentEnd); + LOGGER.debug("[{}] advances time to [{}, {})", context.jobId(), currentStart, currentEnd); } @Override @@ -230,186 +191,10 @@ public void destroy() { @Override public long getEndTime() { - return context.end; + return context.end(); } ChunkedDataExtractorContext getContext() { return context; } - - private class DataSummaryFactory { - - /** - * If there are aggregations, an AggregatedDataSummary object is created. It returns a ScrollingDataSummary otherwise. - * - * By default a DatafeedConfig with aggregations, should already have a manual ChunkingConfig created. - * However, the end user could have specifically set the ChunkingConfig to AUTO, which would not really work for aggregations. - * So, if we need to gather an appropriate chunked time for aggregations, we can utilize the AggregatedDataSummary - * - * @return DataSummary object - */ - private DataSummary buildDataSummary() { - return context.hasAggregations ? newAggregatedDataSummary() : newScrolledDataSummary(); - } - - private DataSummary newScrolledDataSummary() { - SearchRequestBuilder searchRequestBuilder = rangeSearchRequest(); - - SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder); - try { - LOGGER.debug("[{}] Scrolling Data summary response was obtained", context.jobId); - timingStatsReporter.reportSearchDuration(searchResponse.getTook()); - - long earliestTime = 0; - long latestTime = 0; - long totalHits = searchResponse.getHits().getTotalHits().value; - if (totalHits > 0) { - InternalAggregations aggregations = searchResponse.getAggregations(); - Min min = aggregations.get(EARLIEST_TIME); - earliestTime = (long) min.value(); - Max max = aggregations.get(LATEST_TIME); - latestTime = (long) max.value(); - } - return new ScrolledDataSummary(earliestTime, latestTime, totalHits); - } finally { - searchResponse.decRef(); - } - } - - private DataSummary newAggregatedDataSummary() { - // TODO: once RollupSearchAction is changed from indices:admin* to indices:data/read/* this branch is not needed - ActionRequestBuilder searchRequestBuilder = - dataExtractorFactory instanceof RollupDataExtractorFactory ? rollupRangeSearchRequest() : rangeSearchRequest(); - SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder); - try { - LOGGER.debug("[{}] Aggregating Data summary response was obtained", context.jobId); - timingStatsReporter.reportSearchDuration(searchResponse.getTook()); - - InternalAggregations aggregations = searchResponse.getAggregations(); - // This can happen if all the indices the datafeed is searching are deleted after it started. - // Note that unlike the scrolled data summary method above we cannot check for this situation - // by checking for zero hits, because aggregations that work on rollups return zero hits even - // when they retrieve data. - if (aggregations == null) { - return AggregatedDataSummary.noDataSummary(context.histogramInterval); - } - Min min = aggregations.get(EARLIEST_TIME); - Max max = aggregations.get(LATEST_TIME); - return new AggregatedDataSummary(min.value(), max.value(), context.histogramInterval); - } finally { - searchResponse.decRef(); - } - } - - private SearchSourceBuilder rangeSearchBuilder() { - return new SearchSourceBuilder().size(0) - .query(DataExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, currentStart, context.end)) - .runtimeMappings(context.runtimeMappings) - .aggregation(AggregationBuilders.min(EARLIEST_TIME).field(context.timeField)) - .aggregation(AggregationBuilders.max(LATEST_TIME).field(context.timeField)); - } - - private SearchRequestBuilder rangeSearchRequest() { - return new SearchRequestBuilder(client).setIndices(context.indices) - .setIndicesOptions(context.indicesOptions) - .setSource(rangeSearchBuilder()) - .setAllowPartialSearchResults(false) - .setTrackTotalHits(true); - } - - private RollupSearchAction.RequestBuilder rollupRangeSearchRequest() { - SearchRequest searchRequest = new SearchRequest().indices(context.indices) - .indicesOptions(context.indicesOptions) - .allowPartialSearchResults(false) - .source(rangeSearchBuilder()); - return new RollupSearchAction.RequestBuilder(client, searchRequest); - } - } - - private class ScrolledDataSummary implements DataSummary { - - private final long earliestTime; - private final long latestTime; - private final long totalHits; - - private ScrolledDataSummary(long earliestTime, long latestTime, long totalHits) { - this.earliestTime = earliestTime; - this.latestTime = latestTime; - this.totalHits = totalHits; - } - - @Override - public long earliestTime() { - return earliestTime; - } - - @Override - public long getDataTimeSpread() { - return latestTime - earliestTime; - } - - /** - * The heuristic here is that we want a time interval where we expect roughly scrollSize documents - * (assuming data are uniformly spread over time). - * We have totalHits documents over dataTimeSpread (latestTime - earliestTime), we want scrollSize documents over chunk. - * Thus, the interval would be (scrollSize * dataTimeSpread) / totalHits. - * However, assuming this as the chunk span may often lead to half-filled pages or empty searches. - * It is beneficial to take a multiple of that. Based on benchmarking, we set this to 10x. - */ - @Override - public long estimateChunk() { - long dataTimeSpread = getDataTimeSpread(); - if (totalHits <= 0 || dataTimeSpread <= 0) { - return context.end - currentEnd; - } - long estimatedChunk = 10 * (context.scrollSize * getDataTimeSpread()) / totalHits; - return Math.max(estimatedChunk, MIN_CHUNK_SPAN); - } - - @Override - public boolean hasData() { - return totalHits > 0; - } - } - - static class AggregatedDataSummary implements DataSummary { - - private final double earliestTime; - private final double latestTime; - private final long histogramIntervalMillis; - - static AggregatedDataSummary noDataSummary(long histogramInterval) { - // hasData() uses infinity to mean no data - return new AggregatedDataSummary(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY, histogramInterval); - } - - AggregatedDataSummary(double earliestTime, double latestTime, long histogramInterval) { - this.earliestTime = earliestTime; - this.latestTime = latestTime; - this.histogramIntervalMillis = histogramInterval; - } - - /** - * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig} - */ - @Override - public long estimateChunk() { - return DatafeedConfig.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; - } - - @Override - public boolean hasData() { - return (Double.isInfinite(earliestTime) || Double.isInfinite(latestTime)) == false; - } - - @Override - public long earliestTime() { - return (long) earliestTime; - } - - @Override - public long getDataTimeSpread() { - return (long) latestTime - (long) earliestTime; - } - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java index 2989ddb40d370..465c97c38372b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java @@ -6,67 +6,21 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; - -import java.util.List; -import java.util.Map; -import java.util.Objects; - -class ChunkedDataExtractorContext { +record ChunkedDataExtractorContext( + String jobId, + int scrollSize, + long start, + long end, + TimeValue chunkSpan, + TimeAligner timeAligner, + boolean hasAggregations, + Long histogramInterval +) { interface TimeAligner { long alignToFloor(long value); long alignToCeil(long value); } - - final String jobId; - final String timeField; - final String[] indices; - final QueryBuilder query; - final int scrollSize; - final long start; - final long end; - final TimeValue chunkSpan; - final TimeAligner timeAligner; - final Map headers; - final boolean hasAggregations; - final Long histogramInterval; - final IndicesOptions indicesOptions; - final Map runtimeMappings; - - ChunkedDataExtractorContext( - String jobId, - String timeField, - List indices, - QueryBuilder query, - int scrollSize, - long start, - long end, - @Nullable TimeValue chunkSpan, - TimeAligner timeAligner, - Map headers, - boolean hasAggregations, - @Nullable Long histogramInterval, - IndicesOptions indicesOptions, - Map runtimeMappings - ) { - this.jobId = Objects.requireNonNull(jobId); - this.timeField = Objects.requireNonNull(timeField); - this.indices = indices.toArray(new String[indices.size()]); - this.query = Objects.requireNonNull(query); - this.scrollSize = scrollSize; - this.start = start; - this.end = end; - this.chunkSpan = chunkSpan; - this.timeAligner = Objects.requireNonNull(timeAligner); - this.headers = headers; - this.hasAggregations = hasAggregations; - this.histogramInterval = histogramInterval; - this.indicesOptions = Objects.requireNonNull(indicesOptions); - this.runtimeMappings = Objects.requireNonNull(runtimeMappings); - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java index b4141ec632d3b..09414ba58aacb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java @@ -6,14 +6,10 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.utils.Intervals; -import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; @@ -21,57 +17,37 @@ public class ChunkedDataExtractorFactory implements DataExtractorFactory { - private final Client client; private final DatafeedConfig datafeedConfig; - - private final QueryBuilder extraFilters; private final Job job; private final DataExtractorFactory dataExtractorFactory; private final NamedXContentRegistry xContentRegistry; - private final DatafeedTimingStatsReporter timingStatsReporter; public ChunkedDataExtractorFactory( - Client client, DatafeedConfig datafeedConfig, - QueryBuilder extraFilters, Job job, NamedXContentRegistry xContentRegistry, - DataExtractorFactory dataExtractorFactory, - DatafeedTimingStatsReporter timingStatsReporter + DataExtractorFactory dataExtractorFactory ) { - this.client = Objects.requireNonNull(client); this.datafeedConfig = Objects.requireNonNull(datafeedConfig); - this.extraFilters = extraFilters; this.job = Objects.requireNonNull(job); this.dataExtractorFactory = Objects.requireNonNull(dataExtractorFactory); this.xContentRegistry = xContentRegistry; - this.timingStatsReporter = Objects.requireNonNull(timingStatsReporter); } @Override public DataExtractor newExtractor(long start, long end) { - QueryBuilder queryBuilder = datafeedConfig.getParsedQuery(xContentRegistry); - if (extraFilters != null) { - queryBuilder = QueryBuilders.boolQuery().filter(queryBuilder).filter(extraFilters); - } ChunkedDataExtractorContext.TimeAligner timeAligner = newTimeAligner(); ChunkedDataExtractorContext dataExtractorContext = new ChunkedDataExtractorContext( job.getId(), - job.getDataDescription().getTimeField(), - datafeedConfig.getIndices(), - queryBuilder, datafeedConfig.getScrollSize(), timeAligner.alignToCeil(start), timeAligner.alignToFloor(end), datafeedConfig.getChunkingConfig().getTimeSpan(), timeAligner, - datafeedConfig.getHeaders(), datafeedConfig.hasAggregations(), - datafeedConfig.hasAggregations() ? datafeedConfig.getHistogramIntervalMillis(xContentRegistry) : null, - datafeedConfig.getIndicesOptions(), - datafeedConfig.getRuntimeMappings() + datafeedConfig.hasAggregations() ? datafeedConfig.getHistogramIntervalMillis(xContentRegistry) : null ); - return new ChunkedDataExtractor(client, dataExtractorFactory, dataExtractorContext, timingStatsReporter); + return new ChunkedDataExtractor(dataExtractorFactory, dataExtractorContext); } private ChunkedDataExtractorContext.TimeAligner newTimeAligner() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java index 0caa59fae914b..5da89da6b3450 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java @@ -91,7 +91,7 @@ public void destroy() { @Override public long getEndTime() { - return context.end; + return context.queryContext.end; } @Override @@ -103,12 +103,12 @@ public Result next() throws IOException { if (stream.isPresent() == false) { hasNext = false; } - return new Result(new SearchInterval(context.start, context.end), stream); + return new Result(new SearchInterval(context.queryContext.start, context.queryContext.end), stream); } private Optional tryNextStream() throws IOException { try { - return scrollId == null ? Optional.ofNullable(initScroll(context.start)) : Optional.ofNullable(continueScroll()); + return scrollId == null ? Optional.ofNullable(initScroll(context.queryContext.start)) : Optional.ofNullable(continueScroll()); } catch (Exception e) { scrollId = null; if (searchHasShardFailure) { @@ -116,7 +116,7 @@ private Optional tryNextStream() throws IOException { } logger.debug("[{}] Resetting scroll search after shard failure", context.jobId); markScrollAsErrored(); - return Optional.ofNullable(initScroll(lastTimestamp == null ? context.start : lastTimestamp)); + return Optional.ofNullable(initScroll(lastTimestamp == null ? context.queryContext.start : lastTimestamp)); } } @@ -135,14 +135,14 @@ protected InputStream initScroll(long startTimestamp) throws IOException { protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { SearchResponse searchResponse = ClientHelper.executeWithHeaders( - context.headers, + context.queryContext.headers, ClientHelper.ML_ORIGIN, client, searchRequestBuilder::get ); boolean success = false; try { - checkForSkippedClusters(searchResponse); + DataExtractorUtils.checkForSkippedClusters(searchResponse); success = true; } catch (ResourceNotFoundException e) { clearScrollLoggingExceptions(searchResponse.getScrollId()); @@ -158,12 +158,19 @@ protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequest private SearchRequestBuilder buildSearchRequest(long start) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(context.scrollSize) .sort(context.extractedFields.timeField(), SortOrder.ASC) - .query(DataExtractorUtils.wrapInTimeRangeQuery(context.query, context.extractedFields.timeField(), start, context.end)) - .runtimeMappings(context.runtimeMappings); + .query( + DataExtractorUtils.wrapInTimeRangeQuery( + context.queryContext.query, + context.extractedFields.timeField(), + start, + context.queryContext.end + ) + ) + .runtimeMappings(context.queryContext.runtimeMappings); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client).setScroll(SCROLL_TIMEOUT) - .setIndices(context.indices) - .setIndicesOptions(context.indicesOptions) + .setIndices(context.queryContext.indices) + .setIndicesOptions(context.queryContext.indicesOptions) .setAllowPartialSearchResults(false) .setSource(searchSourceBuilder); @@ -228,7 +235,9 @@ private InputStream continueScroll() throws IOException { } logger.debug("[{}] search failed due to SearchPhaseExecutionException. Will attempt again with new scroll", context.jobId); markScrollAsErrored(); - searchResponse = executeSearchRequest(buildSearchRequest(lastTimestamp == null ? context.start : lastTimestamp)); + searchResponse = executeSearchRequest( + buildSearchRequest(lastTimestamp == null ? context.queryContext.start : lastTimestamp) + ); } logger.debug("[{}] Search response was obtained", context.jobId); timingStatsReporter.reportSearchDuration(searchResponse.getTook()); @@ -254,14 +263,14 @@ void markScrollAsErrored() { @SuppressWarnings("HiddenField") protected SearchResponse executeSearchScrollRequest(String scrollId) { SearchResponse searchResponse = ClientHelper.executeWithHeaders( - context.headers, + context.queryContext.headers, ClientHelper.ML_ORIGIN, client, () -> new SearchScrollRequestBuilder(client).setScroll(SCROLL_TIMEOUT).setScrollId(scrollId).get() ); boolean success = false; try { - checkForSkippedClusters(searchResponse); + DataExtractorUtils.checkForSkippedClusters(searchResponse); success = true; } catch (ResourceNotFoundException e) { clearScrollLoggingExceptions(searchResponse.getScrollId()); @@ -294,11 +303,24 @@ private void innerClearScroll(String scrollId) { ClearScrollRequest request = new ClearScrollRequest(); request.addScrollId(scrollId); ClientHelper.executeWithHeaders( - context.headers, + context.queryContext.headers, ClientHelper.ML_ORIGIN, client, () -> client.execute(TransportClearScrollAction.TYPE, request).actionGet() ); } } + + @Override + public DataSummary getSummary() { + SearchRequestBuilder searchRequestBuilder = DataExtractorUtils.getSearchRequestBuilderForSummary(client, context.queryContext); + SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder); + try { + logger.debug("[{}] Scrolling Data summary response was obtained", context.jobId); + timingStatsReporter.reportSearchDuration(searchResponse.getTook()); + return DataExtractorUtils.getDataSummary(searchResponse); + } finally { + searchResponse.decRef(); + } + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java index 58c0c5b485742..776c7c252ffdf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorQueryContext; import java.util.List; import java.util.Map; @@ -18,15 +19,9 @@ class ScrollDataExtractorContext { final String jobId; final TimeBasedExtractedFields extractedFields; - final String[] indices; - final QueryBuilder query; final List scriptFields; final int scrollSize; - final long start; - final long end; - final Map headers; - final IndicesOptions indicesOptions; - final Map runtimeMappings; + final DataExtractorQueryContext queryContext; ScrollDataExtractorContext( String jobId, @@ -41,16 +36,19 @@ class ScrollDataExtractorContext { IndicesOptions indicesOptions, Map runtimeMappings ) { - this.jobId = Objects.requireNonNull(jobId); + this.jobId = jobId; this.extractedFields = Objects.requireNonNull(extractedFields); - this.indices = indices.toArray(new String[indices.size()]); - this.query = Objects.requireNonNull(query); this.scriptFields = Objects.requireNonNull(scriptFields); this.scrollSize = scrollSize; - this.start = start; - this.end = end; - this.headers = headers; - this.indicesOptions = Objects.requireNonNull(indicesOptions); - this.runtimeMappings = Objects.requireNonNull(runtimeMappings); + this.queryContext = new DataExtractorQueryContext( + indices, + query, + extractedFields.timeField(), + start, + end, + headers, + indicesOptions, + runtimeMappings + ); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java index 3fac7c387b12e..e181e1fc86684 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java @@ -293,9 +293,10 @@ public void infer( TimeValue timeout, TrainedModelPrefixStrings.PrefixType prefixType, CancellableTask parentActionTask, + boolean chunkResponse, ActionListener listener ) { - deploymentManager.infer(task, config, input, skipQueue, timeout, prefixType, parentActionTask, listener); + deploymentManager.infer(task, config, input, skipQueue, timeout, prefixType, parentActionTask, chunkResponse, listener); } public Optional modelStats(TrainedModelDeploymentTask task) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java index 1ad7058cb1fdd..d9d6adbba4737 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java @@ -349,6 +349,7 @@ public void infer( TimeValue timeout, TrainedModelPrefixStrings.PrefixType prefixType, CancellableTask parentActionTask, + boolean chunkResponse, ActionListener listener ) { var processContext = getProcessContext(task, listener::onFailure); @@ -368,6 +369,7 @@ public void infer( prefixType, threadPool, parentActionTask, + chunkResponse, listener ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java index 945203c345a3c..0442a37831d70 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.inference.nlp.NlpTask; +import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.NlpTokenizer; import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.TokenizationResult; import org.elasticsearch.xpack.ml.inference.pytorch.results.PyTorchResult; @@ -41,6 +42,7 @@ class InferencePyTorchAction extends AbstractPyTorchAction { @Nullable private final CancellableTask parentActionTask; private final TrainedModelPrefixStrings.PrefixType prefixType; + private final boolean chunkResponse; InferencePyTorchAction( String deploymentId, @@ -52,6 +54,7 @@ class InferencePyTorchAction extends AbstractPyTorchAction { TrainedModelPrefixStrings.PrefixType prefixType, ThreadPool threadPool, @Nullable CancellableTask parentActionTask, + boolean chunkResponse, ActionListener listener ) { super(deploymentId, requestId, timeout, processContext, threadPool, listener); @@ -59,6 +62,7 @@ class InferencePyTorchAction extends AbstractPyTorchAction { this.input = input; this.prefixType = prefixType; this.parentActionTask = parentActionTask; + this.chunkResponse = chunkResponse; } private boolean isCancelled() { @@ -118,8 +122,21 @@ protected void doRun() throws Exception { processor.validateInputs(inputs); assert config instanceof NlpConfig; NlpConfig nlpConfig = (NlpConfig) config; + + int span = nlpConfig.getTokenization().getSpan(); + if (chunkResponse && nlpConfig.getTokenization().getSpan() <= 0) { + // set to special value that means find and use the default for chunking + span = NlpTokenizer.CALC_DEFAULT_SPAN_VALUE; + } + NlpTask.Request request = processor.getRequestBuilder(nlpConfig) - .buildRequest(inputs, requestIdStr, nlpConfig.getTokenization().getTruncate(), nlpConfig.getTokenization().getSpan()); + .buildRequest( + inputs, + requestIdStr, + nlpConfig.getTokenization().getTruncate(), + span, + nlpConfig.getTokenization().maxSequenceLength() + ); logger.debug(() -> format("handling request [%s]", requestIdStr)); // Tokenization is non-trivial, so check for cancellation one last time before sending request to the native process @@ -182,7 +199,11 @@ private void processResult( onFailure("inference task cancelled"); return; } - InferenceResults results = inferenceResultsProcessor.processResult(tokenization, pyTorchResult.inferenceResult()); + InferenceResults results = inferenceResultsProcessor.processResult( + tokenization, + pyTorchResult.inferenceResult(), + this.chunkResponse + ); logger.debug(() -> format("[%s] processed result for request [%s]", getDeploymentId(), getRequestId())); onSuccess(results); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java index 851dd8744d03e..30990e7f88dbb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/TrainedModelDeploymentTask.java @@ -151,6 +151,7 @@ public void infer( TimeValue timeout, TrainedModelPrefixStrings.PrefixType prefixType, CancellableTask parentActionTask, + boolean chunkResponse, ActionListener listener ) { if (inferenceConfigHolder.get() == null) { @@ -172,7 +173,17 @@ public void infer( return; } var updatedConfig = update.isEmpty() ? inferenceConfigHolder.get() : inferenceConfigHolder.get().apply(update); - trainedModelAssignmentNodeService.infer(this, updatedConfig, input, skipQueue, timeout, prefixType, parentActionTask, listener); + trainedModelAssignmentNodeService.infer( + this, + updatedConfig, + input, + skipQueue, + timeout, + prefixType, + parentActionTask, + chunkResponse, + listener + ); } public Optional modelStats() { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessor.java index d171795f86a54..7becab3cc8f48 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessor.java @@ -66,20 +66,22 @@ public NlpTask.RequestBuilder getRequestBuilder(NlpConfig config) { @Override public NlpTask.ResultProcessor getResultProcessor(NlpConfig config) { if (config instanceof FillMaskConfig fillMaskConfig) { - return (tokenization, result) -> processResult( + return (tokenization, result, chunkResults) -> processResult( tokenization, result, tokenizer, fillMaskConfig.getNumTopClasses(), - fillMaskConfig.getResultsField() + fillMaskConfig.getResultsField(), + chunkResults ); } else { - return (tokenization, result) -> processResult( + return (tokenization, result, chunkResults) -> processResult( tokenization, result, tokenizer, FillMaskConfig.DEFAULT_NUM_RESULTS, - DEFAULT_RESULTS_FIELD + DEFAULT_RESULTS_FIELD, + chunkResults ); } } @@ -89,11 +91,15 @@ static InferenceResults processResult( PyTorchInferenceResult pyTorchResult, NlpTokenizer tokenizer, int numResults, - String resultsField + String resultsField, + boolean chunkResults ) { if (tokenization.isEmpty()) { throw new ElasticsearchStatusException("tokenization is empty", RestStatus.INTERNAL_SERVER_ERROR); } + if (chunkResults) { + throw chunkingNotSupportedException(TaskType.NER); + } if (tokenizer.getMaskTokenId().isEmpty()) { throw ExceptionsHelper.conflictStatusException( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java index 599b3c90204ef..3dbf941c8120d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java @@ -179,11 +179,14 @@ record NerResultProcessor(IobTag[] iobMap, String resultsField, boolean ignoreCa } @Override - public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult) { + public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, boolean chunkResult) { if (tokenization.isEmpty()) { throw new ElasticsearchStatusException("no valid tokenization to build result", RestStatus.INTERNAL_SERVER_ERROR); } // TODO - process all results in the batch + if (chunkResult) { + throw chunkingNotSupportedException(TaskType.NER); + } // TODO It might be best to do the soft max after averaging scores for // sub-tokens. If we had a word that is "elastic" which is tokenized to diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NlpTask.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NlpTask.java index 273120bba09ea..06eb493736a89 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NlpTask.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NlpTask.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ml.inference.nlp; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Releasable; @@ -41,11 +42,12 @@ public Processor createProcessor() throws ValidationException { } public interface RequestBuilder { - Request buildRequest(List inputs, String requestId, Tokenization.Truncate truncate, int span) throws IOException; + Request buildRequest(List inputs, String requestId, Tokenization.Truncate truncate, int span, Integer windowSize) + throws IOException; } public interface ResultProcessor { - InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult); + InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, boolean chunkResult); } public abstract static class Processor implements Releasable { @@ -72,6 +74,10 @@ public void close() { public abstract RequestBuilder getRequestBuilder(NlpConfig config); public abstract ResultProcessor getResultProcessor(NlpConfig config); + + static ElasticsearchException chunkingNotSupportedException(TaskType taskType) { + throw chunkingNotSupportedException(TaskType.NER); + } } public record Request(TokenizationResult tokenization, BytesReference processInput) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/PassThroughProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/PassThroughProcessor.java index 417f13964f6cb..acec5c66e720d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/PassThroughProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/PassThroughProcessor.java @@ -44,15 +44,24 @@ public NlpTask.RequestBuilder getRequestBuilder(NlpConfig config) { @Override public NlpTask.ResultProcessor getResultProcessor(NlpConfig config) { - return (tokenization, pyTorchResult) -> processResult(tokenization, pyTorchResult, config.getResultsField()); + return (tokenization, pyTorchResult, chunkResult) -> processResult( + tokenization, + pyTorchResult, + config.getResultsField(), + chunkResult + ); } private static InferenceResults processResult( TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, - String resultsField + String resultsField, + boolean chunkResult ) { - // TODO - process all results in the batch + if (chunkResult) { + throw chunkingNotSupportedException(TaskType.NER); + } + return new PyTorchPassThroughResults( Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), pyTorchResult.getInferenceResult()[0], diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessor.java index a7f7f2bb9f538..0b97b4d0a1ac8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessor.java @@ -66,8 +66,13 @@ public NlpTask.ResultProcessor getResultProcessor(NlpConfig nlpConfig) { record RequestBuilder(NlpTokenizer tokenizer, String question) implements NlpTask.RequestBuilder { @Override - public NlpTask.Request buildRequest(List inputs, String requestId, Tokenization.Truncate truncate, int span) - throws IOException { + public NlpTask.Request buildRequest( + List inputs, + String requestId, + Tokenization.Truncate truncate, + int span, + Integer windowSize + ) throws IOException { if (inputs.size() > 1) { throw ExceptionsHelper.badRequestException("Unable to do question answering on more than one text input at a time"); } @@ -83,7 +88,11 @@ record ResultProcessor(String question, int maxAnswerLength, int numTopClasses, NlpTask.ResultProcessor { @Override - public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult) { + public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, boolean chunkResult) { + if (chunkResult) { + throw chunkingNotSupportedException(TaskType.NER); + } + if (pyTorchResult.getInferenceResult().length < 1) { throw new ElasticsearchStatusException("question answering result has no data", RestStatus.INTERNAL_SERVER_ERROR); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessor.java index 150ac184d246f..3db3e0e999106 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessor.java @@ -58,22 +58,24 @@ public NlpTask.RequestBuilder getRequestBuilder(NlpConfig config) { @Override public NlpTask.ResultProcessor getResultProcessor(NlpConfig config) { if (config instanceof TextClassificationConfig textClassificationConfig) { - return (tokenization, pytorchResult) -> processResult( + return (tokenization, pytorchResult, chunkResult) -> processResult( tokenization, pytorchResult, textClassificationConfig.getNumTopClasses() < 0 ? textClassificationConfig.getClassificationLabels().size() : textClassificationConfig.getNumTopClasses(), textClassificationConfig.getClassificationLabels(), - textClassificationConfig.getResultsField() + textClassificationConfig.getResultsField(), + chunkResult ); } - return (tokenization, pytorchResult) -> processResult( + return (tokenization, pytorchResult, chunkResult) -> processResult( tokenization, pytorchResult, numTopClasses, Arrays.asList(classLabels), - DEFAULT_RESULTS_FIELD + DEFAULT_RESULTS_FIELD, + chunkResult ); } @@ -82,8 +84,13 @@ static InferenceResults processResult( PyTorchInferenceResult pyTorchResult, int numTopClasses, List labels, - String resultsField + String resultsField, + boolean chunkResult ) { + if (chunkResult) { + throw chunkingNotSupportedException(TaskType.NER); + } + if (pyTorchResult.getInferenceResult().length < 1) { throw new ElasticsearchStatusException("Text classification result has no data", RestStatus.INTERNAL_SERVER_ERROR); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextEmbeddingProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextEmbeddingProcessor.java index 453b689d59cc0..22d9294783e7c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextEmbeddingProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextEmbeddingProcessor.java @@ -8,12 +8,14 @@ package org.elasticsearch.xpack.ml.inference.nlp; import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig; import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.NlpTokenizer; import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.TokenizationResult; import org.elasticsearch.xpack.ml.inference.pytorch.results.PyTorchInferenceResult; +import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -43,19 +45,41 @@ public NlpTask.RequestBuilder getRequestBuilder(NlpConfig config) { @Override public NlpTask.ResultProcessor getResultProcessor(NlpConfig config) { - return (tokenization, pyTorchResult) -> processResult(tokenization, pyTorchResult, config.getResultsField()); + return (tokenization, pyTorchResult, chunkResults) -> processResult( + tokenization, + pyTorchResult, + config.getResultsField(), + chunkResults + ); } - private static InferenceResults processResult( + static InferenceResults processResult( TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, - String resultsField + String resultsField, + boolean chunkResults ) { - // TODO - process all results in the batch - return new TextEmbeddingResults( - Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), - pyTorchResult.getInferenceResult()[0][0], - tokenization.anyTruncated() - ); + if (chunkResults) { + var embeddings = new ArrayList(); + for (int i = 0; i < pyTorchResult.getInferenceResult()[0].length; i++) { + int startOffset = tokenization.getTokenization(i).tokens().get(0).get(0).startOffset(); + int lastIndex = tokenization.getTokenization(i).tokens().get(0).size() - 1; + int endOffset = tokenization.getTokenization(i).tokens().get(0).get(lastIndex).endOffset(); + String matchedText = tokenization.getTokenization(i).input().get(0).substring(startOffset, endOffset); + + embeddings.add(new ChunkedTextEmbeddingResults.EmbeddingChunk(matchedText, pyTorchResult.getInferenceResult()[0][i])); + } + return new ChunkedTextEmbeddingResults( + Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), + embeddings, + tokenization.anyTruncated() + ); + } else { + return new TextEmbeddingResults( + Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), + pyTorchResult.getInferenceResult()[0][0], + tokenization.anyTruncated() + ); + } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java index 6483b9d9b3da9..4825faf0d4768 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ml.inference.nlp; import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig; import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.NlpTokenizer; @@ -53,46 +54,50 @@ public NlpTask.RequestBuilder getRequestBuilder(NlpConfig config) { @Override public NlpTask.ResultProcessor getResultProcessor(NlpConfig config) { - return (tokenization, pyTorchResult) -> processResult(tokenization, pyTorchResult, replacementVocab, config.getResultsField()); + return (tokenization, pyTorchResult, chunkResults) -> processResult( + tokenization, + pyTorchResult, + replacementVocab, + config.getResultsField(), + chunkResults + ); } static InferenceResults processResult( TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, Map replacementVocab, - String resultsField + String resultsField, + boolean chunkResults ) { - List weightedTokens; - if (pyTorchResult.getInferenceResult()[0].length == 1) { - weightedTokens = sparseVectorToTokenWeights(pyTorchResult.getInferenceResult()[0][0], tokenization, replacementVocab); - } else { - weightedTokens = multipleSparseVectorsToTokenWeights(pyTorchResult.getInferenceResult()[0], tokenization, replacementVocab); - } - - weightedTokens.sort((t1, t2) -> Float.compare(t2.weight(), t1.weight())); - - return new TextExpansionResults( - Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), - weightedTokens, - tokenization.anyTruncated() - ); - } - - static List multipleSparseVectorsToTokenWeights( - double[][] vector, - TokenizationResult tokenization, - Map replacementVocab - ) { - // reduce to a single 1d array choosing the max value - // in each column and placing that in the first row - for (int i = 1; i < vector.length; i++) { - for (int tokenId = 0; tokenId < vector[i].length; tokenId++) { - if (vector[i][tokenId] > vector[0][tokenId]) { - vector[0][tokenId] = vector[i][tokenId]; - } + if (chunkResults) { + var chunkedResults = new ArrayList(); + + for (int i = 0; i < pyTorchResult.getInferenceResult()[0].length; i++) { + int startOffset = tokenization.getTokenization(i).tokens().get(0).get(0).startOffset(); + int lastIndex = tokenization.getTokenization(i).tokens().get(0).size() - 1; + int endOffset = tokenization.getTokenization(i).tokens().get(0).get(lastIndex).endOffset(); + String matchedText = tokenization.getTokenization(i).input().get(0).substring(startOffset, endOffset); + + var weightedTokens = sparseVectorToTokenWeights(pyTorchResult.getInferenceResult()[0][i], tokenization, replacementVocab); + weightedTokens.sort((t1, t2) -> Float.compare(t2.weight(), t1.weight())); + chunkedResults.add(new ChunkedTextExpansionResults.ChunkedResult(matchedText, weightedTokens)); } + + return new ChunkedTextExpansionResults( + Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), + chunkedResults, + tokenization.anyTruncated() + ); + } else { + var weightedTokens = sparseVectorToTokenWeights(pyTorchResult.getInferenceResult()[0][0], tokenization, replacementVocab); + weightedTokens.sort((t1, t2) -> Float.compare(t2.weight(), t1.weight())); + return new TextExpansionResults( + Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), + weightedTokens, + tokenization.anyTruncated() + ); } - return sparseVectorToTokenWeights(vector[0], tokenization, replacementVocab); } static List sparseVectorToTokenWeights( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessor.java index 0b4f9d8f897e0..525d3adba7457 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessor.java @@ -63,8 +63,13 @@ public NlpTask.ResultProcessor getResultProcessor(NlpConfig nlpConfig) { record RequestBuilder(NlpTokenizer tokenizer, String sequence) implements NlpTask.RequestBuilder { @Override - public NlpTask.Request buildRequest(List inputs, String requestId, Tokenization.Truncate truncate, int span) - throws IOException { + public NlpTask.Request buildRequest( + List inputs, + String requestId, + Tokenization.Truncate truncate, + int span, + Integer windowSize + ) throws IOException { if (inputs.size() > 1) { throw ExceptionsHelper.badRequestException("Unable to do text_similarity on more than one text input at a time"); } @@ -80,7 +85,11 @@ record ResultProcessor(String question, String resultsField, TextSimilarityConfi NlpTask.ResultProcessor { @Override - public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult) { + public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, boolean chunkResult) { + if (chunkResult) { + throw chunkingNotSupportedException(TaskType.NER); + } + if (pyTorchResult.getInferenceResult().length < 1) { throw new ElasticsearchStatusException("text_similarity result has no data", RestStatus.INTERNAL_SERVER_ERROR); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessor.java index a9422e66b16bc..0d3441315700d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessor.java @@ -97,8 +97,13 @@ public NlpTask.ResultProcessor getResultProcessor(NlpConfig nlpConfig) { record RequestBuilder(NlpTokenizer tokenizer, String[] labels, String hypothesisTemplate) implements NlpTask.RequestBuilder { @Override - public NlpTask.Request buildRequest(List inputs, String requestId, Tokenization.Truncate truncate, int span) - throws IOException { + public NlpTask.Request buildRequest( + List inputs, + String requestId, + Tokenization.Truncate truncate, + int span, + Integer windowSize + ) throws IOException { if (inputs.size() > 1) { throw ExceptionsHelper.badRequestException("Unable to do zero-shot classification on more than one text input at a time"); } @@ -129,7 +134,11 @@ record ResultProcessor(int entailmentPos, int contraPos, String[] labels, boolea NlpTask.ResultProcessor { @Override - public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult) { + public InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, boolean chunkResult) { + if (chunkResult) { + throw chunkingNotSupportedException(TaskType.NER); + } + if (pyTorchResult.getInferenceResult().length < 1) { throw new ElasticsearchStatusException("Zero shot classification result has no data", RestStatus.INTERNAL_SERVER_ERROR); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java index 071e77fce196a..45571ea2a8238 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java @@ -169,6 +169,21 @@ boolean isWithSpecialTokens() { return withSpecialTokens; } + @Override + int defaultSpanForChunking(int maxWindowSize) { + return (maxWindowSize - numExtraTokensForSingleSequence()) / 2; + } + + @Override + int getNumExtraTokensForSeqPair() { + return 3; + } + + @Override + int numExtraTokensForSingleSequence() { + return 2; + } + @Override int clsTokenId() { return clsTokenId; @@ -213,19 +228,14 @@ TokenizationResult.TokensBuilder createTokensBuilder(int clsTokenId, int sepToke @Override public NlpTask.RequestBuilder requestBuilder() { - return (inputs, requestId, truncate, span) -> buildTokenizationResult( + return (inputs, requestId, truncate, span, windowSize) -> buildTokenizationResult( IntStream.range(0, inputs.size()) .boxed() - .flatMap(seqId -> tokenize(inputs.get(seqId), truncate, span, seqId).stream()) + .flatMap(seqId -> tokenize(inputs.get(seqId), truncate, span, seqId, windowSize).stream()) .collect(Collectors.toList()) ).buildRequest(requestId, truncate); } - @Override - int getNumExtraTokensForSeqPair() { - return 3; - } - @Override public InnerTokenization innerTokenize(String seq) { List tokenPositionMap = new ArrayList<>(); @@ -248,10 +258,6 @@ public void close() { wordPieceAnalyzer.close(); } - public int getMaxSequenceLength() { - return maxSequenceLength; - } - public static Builder builder(List vocab, Tokenization tokenization) { return new Builder(vocab, tokenization); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java index 225ee75b596be..5014eb269b081 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java @@ -33,6 +33,9 @@ * Base tokenization class for NLP models */ public abstract class NlpTokenizer implements Releasable { + + public static final int CALC_DEFAULT_SPAN_VALUE = -2; + abstract int clsTokenId(); abstract int sepTokenId(); @@ -41,8 +44,12 @@ public abstract class NlpTokenizer implements Releasable { abstract boolean isWithSpecialTokens(); + abstract int numExtraTokensForSingleSequence(); + abstract int getNumExtraTokensForSeqPair(); + abstract int defaultSpanForChunking(int maxWindowSize); + public abstract TokenizationResult buildTokenizationResult(List tokenizations); /** @@ -54,35 +61,51 @@ public abstract class NlpTokenizer implements Releasable { * each input string grouped into a {@link Tokenization}. * * @param seq Text to tokenize + * @param truncate + * @param span + * @param sequenceId + * @param windowSize * @return A list of {@link Tokenization} */ - public List tokenize(String seq, Tokenization.Truncate truncate, int span, int sequenceId) { + public final List tokenize( + String seq, + Tokenization.Truncate truncate, + int span, + int sequenceId, + Integer windowSize + ) { + if (windowSize == null) { + windowSize = maxSequenceLength(); + } var innerResult = innerTokenize(seq); List tokenIds = innerResult.tokens(); List tokenPositionMap = innerResult.tokenPositionMap(); - int numTokens = isWithSpecialTokens() ? tokenIds.size() + 2 : tokenIds.size(); + int numTokens = isWithSpecialTokens() ? tokenIds.size() + numExtraTokensForSingleSequence() : tokenIds.size(); boolean isTruncated = false; - if (numTokens > maxSequenceLength()) { + if (numTokens > windowSize) { switch (truncate) { case FIRST, SECOND -> { isTruncated = true; - tokenIds = tokenIds.subList(0, isWithSpecialTokens() ? maxSequenceLength() - 2 : maxSequenceLength()); - tokenPositionMap = tokenPositionMap.subList(0, isWithSpecialTokens() ? maxSequenceLength() - 2 : maxSequenceLength()); + tokenIds = tokenIds.subList(0, isWithSpecialTokens() ? windowSize - numExtraTokensForSingleSequence() : windowSize); + tokenPositionMap = tokenPositionMap.subList( + 0, + isWithSpecialTokens() ? windowSize - numExtraTokensForSingleSequence() : windowSize + ); } case NONE -> { if (span == -1) { throw ExceptionsHelper.badRequestException( "Input too large. The tokenized input length [{}] exceeds the maximum sequence length [{}]", numTokens, - maxSequenceLength() + windowSize ); } } } } - if (numTokens <= maxSequenceLength() || span == -1) { + if (numTokens <= windowSize || span == -1) { return List.of( createTokensBuilder(clsTokenId(), sepTokenId(), isWithSpecialTokens()).addSequence( tokenIds.stream().map(DelimitedToken.Encoded::getEncoding).collect(Collectors.toList()), @@ -91,13 +114,17 @@ public List tokenize(String seq, Tokenization.Truncat ); } + if (span == CALC_DEFAULT_SPAN_VALUE) { + span = defaultSpanForChunking(windowSize); + } + List toReturn = new ArrayList<>(); int splitEndPos = 0; int splitStartPos = 0; int spanPrev = -1; while (splitEndPos < tokenIds.size()) { splitEndPos = Math.min( - splitStartPos + (isWithSpecialTokens() ? maxSequenceLength() - 2 : maxSequenceLength()), + splitStartPos + (isWithSpecialTokens() ? windowSize - numExtraTokensForSingleSequence() : windowSize), tokenIds.size() ); // Make sure we do not end on a word diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java index b146abc35af33..d604b52a55cc4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java @@ -106,6 +106,16 @@ int getNumExtraTokensForSeqPair() { return 4; } + @Override + int defaultSpanForChunking(int maxWindowSize) { + return (maxWindowSize - numExtraTokensForSingleSequence()) / 2; + } + + @Override + int numExtraTokensForSingleSequence() { + return 2; + } + @Override int clsTokenId() { return clsTokenId; @@ -131,10 +141,10 @@ public TokenizationResult buildTokenizationResult(List buildTokenizationResult( + return (inputs, requestId, truncate, span, windowSize) -> buildTokenizationResult( IntStream.range(0, inputs.size()) .boxed() - .flatMap(seqId -> tokenize(inputs.get(seqId), truncate, span, seqId).stream()) + .flatMap(seqId -> tokenize(inputs.get(seqId), truncate, span, seqId, windowSize).stream()) .collect(Collectors.toList()) ).buildRequest(requestId, truncate); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java index 2f934a3996321..3c7d54cd547bf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java @@ -101,6 +101,16 @@ int getNumExtraTokensForSeqPair() { return 4; } + @Override + int defaultSpanForChunking(int maxWindowSize) { + return (maxWindowSize - numExtraTokensForSingleSequence()) / 2; + } + + @Override + int numExtraTokensForSingleSequence() { + return 2; + } + @Override int clsTokenId() { return clsTokenId; @@ -126,10 +136,10 @@ public TokenizationResult buildTokenizationResult(List buildTokenizationResult( + return (inputs, requestId, truncate, span, windowSize) -> buildTokenizationResult( IntStream.range(0, inputs.size()) .boxed() - .flatMap(seqId -> tokenize(inputs.get(seqId), truncate, span, seqId).stream()) + .flatMap(seqId -> tokenize(inputs.get(seqId), truncate, span, seqId, windowSize).stream()) .collect(Collectors.toList()) ).buildRequest(requestId, truncate); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java index 28cdb31700a29..c35b9da7b2bd2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java @@ -222,7 +222,7 @@ public void testAnomalyDetectionOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, true, false, false, false)); try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { - List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null); + List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, hasItem(instanceOf(RestGetJobsAction.class))); assertThat(restHandlers, not(hasItem(instanceOf(RestGetTrainedModelsAction.class)))); @@ -242,7 +242,7 @@ public void testDataFrameAnalyticsOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, true, false, false)); try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { - List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null); + List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, not(hasItem(instanceOf(RestGetJobsAction.class)))); assertThat(restHandlers, hasItem(instanceOf(RestGetTrainedModelsAction.class))); @@ -262,7 +262,7 @@ public void testNlpOnly() throws IOException { Settings settings = Settings.builder().put("path.home", createTempDir()).build(); MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, false, true, false)); try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) { - List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null); + List restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null, null); assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class))); assertThat(restHandlers, not(hasItem(instanceOf(RestGetJobsAction.class)))); assertThat(restHandlers, hasItem(instanceOf(RestGetTrainedModelsAction.class))); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java index 9a76eb5f2b936..2a086457ba755 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java @@ -50,8 +50,8 @@ public void testNewExtractor_GivenAlignedTimes() { AggregationDataExtractor dataExtractor = (AggregationDataExtractor) factory.newExtractor(2000, 5000); - assertThat(dataExtractor.getContext().start, equalTo(2000L)); - assertThat(dataExtractor.getContext().end, equalTo(5000L)); + assertThat(dataExtractor.getContext().queryContext.start, equalTo(2000L)); + assertThat(dataExtractor.getContext().queryContext.end, equalTo(5000L)); } public void testNewExtractor_GivenNonAlignedTimes() { @@ -59,8 +59,8 @@ public void testNewExtractor_GivenNonAlignedTimes() { AggregationDataExtractor dataExtractor = (AggregationDataExtractor) factory.newExtractor(3980, 9200); - assertThat(dataExtractor.getContext().start, equalTo(4000L)); - assertThat(dataExtractor.getContext().end, equalTo(9000L)); + assertThat(dataExtractor.getContext().queryContext.start, equalTo(4000L)); + assertThat(dataExtractor.getContext().queryContext.end, equalTo(9000L)); } private AggregationDataExtractorFactory createFactory(long histogramInterval) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java index 5b2cd8f78d02e..02e33695ff7e2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java @@ -6,28 +6,35 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; +import org.elasticsearch.search.aggregations.metrics.Max; +import org.elasticsearch.search.aggregations.metrics.Min; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.datafeed.SearchInterval; import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; -import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter.DatafeedTimingStatsPersister; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.Term; import org.junit.Before; +import org.mockito.ArgumentCaptor; import java.io.BufferedReader; import java.io.IOException; @@ -50,14 +57,17 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.stringContainsInOrder; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class AggregationDataExtractorTests extends ESTestCase { - private Client testClient; - private List capturedSearchRequests; + private Client client; private String jobId; private String timeField; private Set fields; @@ -67,37 +77,12 @@ public class AggregationDataExtractorTests extends ESTestCase { private DatafeedTimingStatsReporter timingStatsReporter; private Map runtimeMappings; - private class TestDataExtractor extends AggregationDataExtractor { - - private SearchResponse nextResponse; - private SearchPhaseExecutionException ex; - - TestDataExtractor(long start, long end) { - super(testClient, createContext(start, end), timingStatsReporter); - } - - @Override - protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { - capturedSearchRequests.add(searchRequestBuilder); - if (ex != null) { - throw ex; - } - return nextResponse; - } - - void setNextResponse(SearchResponse searchResponse) { - nextResponse = searchResponse; - } - - void setNextResponseToError(SearchPhaseExecutionException ex) { - this.ex = ex; - } - } - @Before public void setUpTests() { - testClient = mock(Client.class); - capturedSearchRequests = new ArrayList<>(); + client = mock(Client.class); + when(client.threadPool()).thenReturn(mock(ThreadPool.class)); + when(client.threadPool().getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); + jobId = "test-job"; timeField = "time"; fields = new HashSet<>(); @@ -115,7 +100,7 @@ public void setUpTests() { ) ); runtimeMappings = Collections.emptyMap(); - timingStatsReporter = new DatafeedTimingStatsReporter(new DatafeedTimingStats(jobId), mock(DatafeedTimingStatsPersister.class)); + timingStatsReporter = mock(DatafeedTimingStatsReporter.class); } public void testExtraction() throws IOException { @@ -139,10 +124,11 @@ public void testExtraction() throws IOException { ) ); - TestDataExtractor extractor = new TestDataExtractor(1000L, 4000L); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 4000L), timingStatsReporter); - SearchResponse response = createSearchResponse("time", histogramBuckets); - extractor.setNextResponse(response); + ArgumentCaptor searchRequestCaptor = ArgumentCaptor.forClass(SearchRequest.class); + ActionFuture searchResponse = toActionFuture(createSearchResponse("time", histogramBuckets)); + when(client.execute(eq(TransportSearchAction.TYPE), searchRequestCaptor.capture())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); DataExtractor.Result result = extractor.next(); @@ -156,9 +142,8 @@ public void testExtraction() throws IOException { {"time":3999,"airline":"b","responsetime":32.0,"doc_count":3}"""; assertThat(asString(stream.get()), equalTo(expectedStream)); assertThat(extractor.hasNext(), is(false)); - assertThat(capturedSearchRequests.size(), equalTo(1)); - String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + String searchRequest = searchRequestCaptor.getValue().toString().replaceAll("\\s", ""); assertThat(searchRequest, containsString("\"size\":0")); assertThat( searchRequest, @@ -175,45 +160,47 @@ public void testExtraction() throws IOException { } public void testExtractionGivenResponseHasNullAggs() throws IOException { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 2000L), timingStatsReporter); - SearchResponse response = createSearchResponse(null); - extractor.setNextResponse(response); + ActionFuture searchResponse = toActionFuture(createSearchResponse(null)); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); - assertThat(capturedSearchRequests.size(), equalTo(1)); + verify(client).execute(eq(TransportSearchAction.TYPE), any()); } public void testExtractionGivenResponseHasEmptyAggs() throws IOException { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 2000L), timingStatsReporter); + InternalAggregations emptyAggs = AggregationTestUtils.createAggs(Collections.emptyList()); - SearchResponse response = createSearchResponse(emptyAggs); - extractor.setNextResponse(response); + ActionFuture searchResponse = toActionFuture(createSearchResponse(emptyAggs)); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); - assertThat(capturedSearchRequests.size(), equalTo(1)); + verify(client).execute(eq(TransportSearchAction.TYPE), any()); } public void testExtractionGivenResponseHasEmptyHistogramAgg() throws IOException { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); - SearchResponse response = createSearchResponse("time", Collections.emptyList()); - extractor.setNextResponse(response); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 2000L), timingStatsReporter); + + ActionFuture searchResponse = toActionFuture(createSearchResponse("time", Collections.emptyList())); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); - assertThat(capturedSearchRequests.size(), equalTo(1)); + verify(client).execute(eq(TransportSearchAction.TYPE), any()); } public void testExtractionGivenResponseHasMultipleTopLevelAggs() { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 2000L), timingStatsReporter); InternalHistogram histogram1 = mock(InternalHistogram.class); when(histogram1.getName()).thenReturn("hist_1"); @@ -221,8 +208,8 @@ public void testExtractionGivenResponseHasMultipleTopLevelAggs() { when(histogram2.getName()).thenReturn("hist_2"); InternalAggregations aggs = AggregationTestUtils.createAggs(Arrays.asList(histogram1, histogram2)); - SearchResponse response = createSearchResponse(aggs); - extractor.setNextResponse(response); + ActionFuture searchResponse = toActionFuture(createSearchResponse(aggs)); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, extractor::next); @@ -230,9 +217,10 @@ public void testExtractionGivenResponseHasMultipleTopLevelAggs() { } public void testExtractionGivenCancelBeforeNext() { - TestDataExtractor extractor = new TestDataExtractor(1000L, 4000L); - SearchResponse response = createSearchResponse("time", Collections.emptyList()); - extractor.setNextResponse(response); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 4000L), timingStatsReporter); + + ActionFuture searchResponse = toActionFuture(createSearchResponse("time", Collections.emptyList())); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); extractor.cancel(); assertThat(extractor.hasNext(), is(false)); @@ -256,10 +244,10 @@ public void testExtractionGivenCancelHalfWay() throws IOException { timestamp += 1000L; } - TestDataExtractor extractor = new TestDataExtractor(1000L, timestamp + 1); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, timestamp + 1), timingStatsReporter); - SearchResponse response = createSearchResponse("time", histogramBuckets); - extractor.setNextResponse(response); + ActionFuture searchResponse = toActionFuture(createSearchResponse("time", histogramBuckets)); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); assertThat(countMatches('{', asString(extractor.next().data().get())), equalTo(2400L)); @@ -277,23 +265,57 @@ public void testExtractionGivenCancelHalfWay() throws IOException { ); timestamp += 1000L; } - response = createSearchResponse("time", histogramBuckets); - extractor.setNextResponse(response); + searchResponse = toActionFuture(createSearchResponse("time", histogramBuckets)); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); extractor.cancel(); assertThat(extractor.hasNext(), is(false)); assertThat(extractor.isCancelled(), is(true)); - assertThat(capturedSearchRequests.size(), equalTo(1)); + verify(client).execute(eq(TransportSearchAction.TYPE), any()); } public void testExtractionGivenSearchResponseHasError() { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); - extractor.setNextResponseToError(new SearchPhaseExecutionException("phase 1", "boom", ShardSearchFailure.EMPTY_ARRAY)); + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 2000L), timingStatsReporter); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenThrow( + new SearchPhaseExecutionException("phase 1", "boom", ShardSearchFailure.EMPTY_ARRAY) + ); assertThat(extractor.hasNext(), is(true)); expectThrows(SearchPhaseExecutionException.class, extractor::next); } + public void testGetSummary() { + AggregationDataExtractor extractor = new AggregationDataExtractor(client, createContext(1000L, 2300L), timingStatsReporter); + + ArgumentCaptor searchRequestCaptor = ArgumentCaptor.forClass(SearchRequest.class); + ActionFuture searchResponse = toActionFuture(createSummaryResponse(1001L, 2299L, 10L)); + when(client.execute(eq(TransportSearchAction.TYPE), searchRequestCaptor.capture())).thenReturn(searchResponse); + + DataExtractor.DataSummary summary = extractor.getSummary(); + assertThat(summary.earliestTime(), equalTo(1001L)); + assertThat(summary.latestTime(), equalTo(2299L)); + assertThat(summary.totalHits(), equalTo(10L)); + + String searchRequest = searchRequestCaptor.getValue().toString().replaceAll("\\s", ""); + assertThat(searchRequest, containsString("\"size\":0")); + assertThat( + searchRequest, + containsString( + "\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," + + "{\"range\":{\"time\":{\"gte\":1000,\"lt\":2300," + + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]" + ) + ); + assertThat( + searchRequest, + containsString( + "\"aggregations\":{\"earliest_time\":{\"min\":{\"field\":\"time\"}}," + "\"latest_time\":{\"max\":{\"field\":\"time\"}}}}" + ) + ); + assertThat(searchRequest, not(containsString("\"track_total_hits\":false"))); + assertThat(searchRequest, not(containsString("\"sort\""))); + } + private AggregationDataExtractorContext createContext(long start, long end) { return new AggregationDataExtractorContext( jobId, @@ -311,7 +333,13 @@ private AggregationDataExtractorContext createContext(long start, long end) { ); } - @SuppressWarnings("unchecked") + private ActionFuture toActionFuture(T t) { + @SuppressWarnings("unchecked") + ActionFuture future = (ActionFuture) mock(ActionFuture.class); + when(future.actionGet()).thenReturn(t); + return future; + } + private SearchResponse createSearchResponse(String histogramName, List histogramBuckets) { InternalHistogram histogram = mock(InternalHistogram.class); when(histogram.getName()).thenReturn(histogramName); @@ -330,6 +358,17 @@ private SearchResponse createSearchResponse(InternalAggregations aggregations) { return searchResponse; } + private SearchResponse createSummaryResponse(long start, long end, long totalHits) { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn( + new SearchHits(SearchHits.EMPTY, new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), 1) + ); + when(searchResponse.getAggregations()).thenReturn( + InternalAggregations.from(List.of(new Min("earliest_time", start, null, null), new Max("latest_time", end, null, null))) + ); + return searchResponse; + } + private static String asString(InputStream inputStream) throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { return reader.lines().collect(Collectors.joining("\n")); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorTests.java index 6cc432dd4831f..5b9370d53e26e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorTests.java @@ -6,13 +6,16 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; -import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryBuilder; @@ -26,12 +29,12 @@ import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.datafeed.SearchInterval; import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; -import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter.DatafeedTimingStatsPersister; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; import org.junit.Before; +import org.mockito.ArgumentCaptor; import java.io.BufferedReader; import java.io.IOException; @@ -55,13 +58,16 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.stringContainsInOrder; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class CompositeAggregationDataExtractorTests extends ESTestCase { - private Client testClient; - private List capturedSearchRequests; + private Client client; private String jobId; private String timeField; private Set fields; @@ -72,37 +78,12 @@ public class CompositeAggregationDataExtractorTests extends ESTestCase { private AggregatedSearchRequestBuilder aggregatedSearchRequestBuilder; private Map runtimeMappings; - private class TestDataExtractor extends CompositeAggregationDataExtractor { - - private SearchResponse nextResponse; - private SearchPhaseExecutionException ex; - - TestDataExtractor(long start, long end) { - super(compositeAggregationBuilder, testClient, createContext(start, end), timingStatsReporter, aggregatedSearchRequestBuilder); - } - - @Override - protected SearchResponse executeSearchRequest(ActionRequestBuilder searchRequestBuilder) { - capturedSearchRequests.add(searchRequestBuilder.request()); - if (ex != null) { - throw ex; - } - return nextResponse; - } - - void setNextResponse(SearchResponse searchResponse) { - nextResponse = searchResponse; - } - - void setNextResponseToError(SearchPhaseExecutionException ex) { - this.ex = ex; - } - } - @Before public void setUpTests() { - testClient = mock(Client.class); - capturedSearchRequests = new ArrayList<>(); + client = mock(Client.class); + when(client.threadPool()).thenReturn(mock(ThreadPool.class)); + when(client.threadPool().getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); + jobId = "test-job"; timeField = "time"; fields = new HashSet<>(); @@ -120,8 +101,8 @@ public void setUpTests() { .subAggregation(AggregationBuilders.max("time").field("time")) .subAggregation(AggregationBuilders.avg("responsetime").field("responsetime")); runtimeMappings = Collections.emptyMap(); - timingStatsReporter = new DatafeedTimingStatsReporter(new DatafeedTimingStats(jobId), mock(DatafeedTimingStatsPersister.class)); - aggregatedSearchRequestBuilder = (searchSourceBuilder) -> new SearchRequestBuilder(testClient).setSource(searchSourceBuilder) + timingStatsReporter = mock(DatafeedTimingStatsReporter.class); + aggregatedSearchRequestBuilder = (searchSourceBuilder) -> new SearchRequestBuilder(client).setSource(searchSourceBuilder) .setAllowPartialSearchResults(false) .setIndices(indices.toArray(String[]::new)); } @@ -159,10 +140,19 @@ public void testExtraction() throws IOException { ) ); - TestDataExtractor extractor = new TestDataExtractor(1000L, 4000L); + CompositeAggregationDataExtractor extractor = new CompositeAggregationDataExtractor( + compositeAggregationBuilder, + client, + createContext(1000L, 4000L), + timingStatsReporter, + aggregatedSearchRequestBuilder + ); - SearchResponse response = createSearchResponse("buckets", compositeBucket, Map.of("time_bucket", 4000L, "airline", "d")); - extractor.setNextResponse(response); + ArgumentCaptor searchRequestCaptor = ArgumentCaptor.forClass(SearchRequest.class); + ActionFuture searchResponse = toActionFuture( + createSearchResponse("buckets", compositeBucket, Map.of("time_bucket", 4000L, "airline", "d")) + ); + when(client.execute(eq(TransportSearchAction.TYPE), searchRequestCaptor.capture())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); DataExtractor.Result result = extractor.next(); @@ -175,9 +165,8 @@ public void testExtraction() throws IOException { {"airline":"c","time":3999,"responsetime":31.0,"doc_count":4} \ {"airline":"b","time":3999,"responsetime":32.0,"doc_count":3}"""; assertThat(asString(stream.get()), equalTo(expectedStream)); - assertThat(capturedSearchRequests.size(), equalTo(1)); - String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + String searchRequest = searchRequestCaptor.getValue().toString().replaceAll("\\s", ""); assertThat(searchRequest, containsString("\"size\":0")); assertThat( searchRequest, @@ -194,35 +183,57 @@ public void testExtraction() throws IOException { } public void testExtractionGivenResponseHasNullAggs() throws IOException { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + CompositeAggregationDataExtractor extractor = new CompositeAggregationDataExtractor( + compositeAggregationBuilder, + client, + createContext(1000L, 2000L), + timingStatsReporter, + aggregatedSearchRequestBuilder + ); - SearchResponse response = createSearchResponse(null); - extractor.setNextResponse(response); + ActionFuture searchResponse = toActionFuture(createSearchResponse(null)); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); - assertThat(capturedSearchRequests.size(), equalTo(1)); + verify(client).execute(eq(TransportSearchAction.TYPE), any()); } public void testExtractionGivenResponseHasEmptyAggs() throws IOException { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + CompositeAggregationDataExtractor extractor = new CompositeAggregationDataExtractor( + compositeAggregationBuilder, + client, + createContext(1000L, 2000L), + timingStatsReporter, + aggregatedSearchRequestBuilder + ); + InternalAggregations emptyAggs = AggregationTestUtils.createAggs(Collections.emptyList()); - SearchResponse response = createSearchResponse(emptyAggs); - extractor.setNextResponse(response); + ActionFuture searchResponse = toActionFuture(createSearchResponse(emptyAggs)); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); - assertThat(capturedSearchRequests.size(), equalTo(1)); + verify(client).execute(eq(TransportSearchAction.TYPE), any()); } public void testExtractionGivenCancelBeforeNext() { - TestDataExtractor extractor = new TestDataExtractor(1000L, 4000L); - SearchResponse response = createSearchResponse("time", Collections.emptyList(), Collections.emptyMap()); - extractor.setNextResponse(response); + CompositeAggregationDataExtractor extractor = new CompositeAggregationDataExtractor( + compositeAggregationBuilder, + client, + createContext(1000L, 4000L), + timingStatsReporter, + aggregatedSearchRequestBuilder + ); + + ActionFuture searchResponse = toActionFuture( + createSearchResponse("time", Collections.emptyList(), Collections.emptyMap()) + ); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); extractor.cancel(); // Composite aggs should be true because we need to make sure the first search has occurred or not @@ -245,10 +256,19 @@ public void testExtractionCancelOnFirstPage() throws IOException { ); } - TestDataExtractor extractor = new TestDataExtractor(1000L, timestamp + 1000 + 1); + CompositeAggregationDataExtractor extractor = new CompositeAggregationDataExtractor( + compositeAggregationBuilder, + client, + createContext(1000L, timestamp + 1000 + 1), + timingStatsReporter, + aggregatedSearchRequestBuilder + ); + + ActionFuture searchResponse = toActionFuture( + createSearchResponse("buckets", buckets, Map.of("time_bucket", 1000L, "airline", "d")) + ); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); - SearchResponse response = createSearchResponse("buckets", buckets, Map.of("time_bucket", 1000L, "airline", "d")); - extractor.setNextResponse(response); extractor.cancel(); // We should have next right now as we have not yet determined if we have handled a page or not assertThat(extractor.hasNext(), is(true)); @@ -274,10 +294,18 @@ public void testExtractionGivenCancelHalfWay() throws IOException { ); } - TestDataExtractor extractor = new TestDataExtractor(1000L, timestamp + 1000 + 1); + CompositeAggregationDataExtractor extractor = new CompositeAggregationDataExtractor( + compositeAggregationBuilder, + client, + createContext(1000L, timestamp + 1000 + 1), + timingStatsReporter, + aggregatedSearchRequestBuilder + ); - SearchResponse response = createSearchResponse("buckets", buckets, Map.of("time_bucket", 1000L, "airline", "d")); - extractor.setNextResponse(response); + ActionFuture searchResponse = toActionFuture( + createSearchResponse("buckets", buckets, Map.of("time_bucket", 1000L, "airline", "d")) + ); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); assertThat(extractor.hasNext(), is(true)); assertThat(countMatches('{', asString(extractor.next().data().get())), equalTo(10L)); @@ -305,8 +333,10 @@ public void testExtractionGivenCancelHalfWay() throws IOException { ) ); } - response = createSearchResponse("buckets", buckets, Map.of("time_bucket", 3000L, "airline", "a")); - extractor.setNextResponse(response); + + searchResponse = toActionFuture(createSearchResponse("buckets", buckets, Map.of("time_bucket", 3000L, "airline", "a"))); + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenReturn(searchResponse); + extractor.cancel(); assertThat(extractor.hasNext(), is(true)); assertThat(extractor.isCancelled(), is(true)); @@ -315,12 +345,22 @@ public void testExtractionGivenCancelHalfWay() throws IOException { // Once we have handled the 6 remaining in that time bucket, we shouldn't finish the page and the extractor should end assertThat(extractor.hasNext(), is(false)); - assertThat(capturedSearchRequests.size(), equalTo(2)); + + verify(client, times(2)).execute(eq(TransportSearchAction.TYPE), any()); } public void testExtractionGivenSearchResponseHasError() { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); - extractor.setNextResponseToError(new SearchPhaseExecutionException("phase 1", "boom", ShardSearchFailure.EMPTY_ARRAY)); + CompositeAggregationDataExtractor extractor = new CompositeAggregationDataExtractor( + compositeAggregationBuilder, + client, + createContext(1000L, 2000L), + timingStatsReporter, + aggregatedSearchRequestBuilder + ); + + when(client.execute(eq(TransportSearchAction.TYPE), any())).thenThrow( + new SearchPhaseExecutionException("phase 1", "boom", ShardSearchFailure.EMPTY_ARRAY) + ); assertThat(extractor.hasNext(), is(true)); expectThrows(SearchPhaseExecutionException.class, extractor::next); @@ -344,7 +384,13 @@ private CompositeAggregationDataExtractorContext createContext(long start, long ); } - @SuppressWarnings("unchecked") + private ActionFuture toActionFuture(T t) { + @SuppressWarnings("unchecked") + ActionFuture future = (ActionFuture) mock(ActionFuture.class); + when(future.actionGet()).thenReturn(t); + return future; + } + private SearchResponse createSearchResponse( String aggName, List buckets, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java index a7260d34a0136..878f49dbe77fe 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -18,7 +17,6 @@ import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; import org.junit.Before; @@ -31,9 +29,7 @@ public class ChunkedDataExtractorFactoryTests extends ESTestCase { - private Client client; private DataExtractorFactory dataExtractorFactory; - private DatafeedTimingStatsReporter timingStatsReporter; @Override protected NamedXContentRegistry xContentRegistry() { @@ -43,9 +39,7 @@ protected NamedXContentRegistry xContentRegistry() { @Before public void setUpMocks() { - client = mock(Client.class); dataExtractorFactory = mock(DataExtractorFactory.class); - timingStatsReporter = mock(DatafeedTimingStatsReporter.class); } public void testNewExtractor_GivenAlignedTimes() { @@ -53,8 +47,8 @@ public void testNewExtractor_GivenAlignedTimes() { ChunkedDataExtractor dataExtractor = (ChunkedDataExtractor) factory.newExtractor(2000, 5000); - assertThat(dataExtractor.getContext().start, equalTo(2000L)); - assertThat(dataExtractor.getContext().end, equalTo(5000L)); + assertThat(dataExtractor.getContext().start(), equalTo(2000L)); + assertThat(dataExtractor.getContext().end(), equalTo(5000L)); } public void testNewExtractor_GivenNonAlignedTimes() { @@ -62,8 +56,8 @@ public void testNewExtractor_GivenNonAlignedTimes() { ChunkedDataExtractor dataExtractor = (ChunkedDataExtractor) factory.newExtractor(3980, 9200); - assertThat(dataExtractor.getContext().start, equalTo(4000L)); - assertThat(dataExtractor.getContext().end, equalTo(9000L)); + assertThat(dataExtractor.getContext().start(), equalTo(4000L)); + assertThat(dataExtractor.getContext().end(), equalTo(9000L)); } public void testIntervalTimeAligner() { @@ -111,13 +105,10 @@ private ChunkedDataExtractorFactory createFactory(long histogramInterval) { datafeedConfigBuilder.setParsedAggregations(aggs); datafeedConfigBuilder.setIndices(Arrays.asList("my_index")); return new ChunkedDataExtractorFactory( - client, datafeedConfigBuilder.build(), - null, jobBuilder.build(new Date()), xContentRegistry(), - dataExtractorFactory, - timingStatsReporter + dataExtractorFactory ); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java index 7c8d2572461d4..ce6cf92d4bd51 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java @@ -6,29 +6,13 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; -import org.apache.lucene.search.TotalHits; -import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.aggregations.metrics.Max; -import org.elasticsearch.search.aggregations.metrics.Min; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; import org.elasticsearch.xpack.core.ml.datafeed.SearchInterval; -import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; -import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter.DatafeedTimingStatsPersister; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor.DataSummary; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; import org.junit.Before; import org.mockito.Mockito; @@ -36,100 +20,62 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ChunkedDataExtractorTests extends ESTestCase { - private Client client; - private List capturedSearchRequests; private String jobId; - private String timeField; - private List indices; - private QueryBuilder query; private int scrollSize; private TimeValue chunkSpan; private DataExtractorFactory dataExtractorFactory; - private DatafeedTimingStatsReporter timingStatsReporter; - - private class TestDataExtractor extends ChunkedDataExtractor { - - private SearchResponse nextResponse; - private SearchPhaseExecutionException ex; - - TestDataExtractor(long start, long end) { - super(client, dataExtractorFactory, createContext(start, end), timingStatsReporter); - } - - TestDataExtractor(long start, long end, boolean hasAggregations, Long histogramInterval) { - super(client, dataExtractorFactory, createContext(start, end, hasAggregations, histogramInterval), timingStatsReporter); - } - - @Override - protected SearchResponse executeSearchRequest(ActionRequestBuilder searchRequestBuilder) { - capturedSearchRequests.add(searchRequestBuilder.request()); - if (ex != null) { - throw ex; - } - return nextResponse; - } - - void setNextResponse(SearchResponse searchResponse) { - nextResponse = searchResponse; - } - - void setNextResponseToError(SearchPhaseExecutionException ex) { - this.ex = ex; - } - } @Before public void setUpTests() { - client = mock(Client.class); - capturedSearchRequests = new ArrayList<>(); jobId = "test-job"; - timeField = "time"; - indices = Arrays.asList("index-1", "index-2"); scrollSize = 1000; chunkSpan = null; dataExtractorFactory = mock(DataExtractorFactory.class); - timingStatsReporter = new DatafeedTimingStatsReporter(new DatafeedTimingStats(jobId), mock(DatafeedTimingStatsPersister.class)); } public void testExtractionGivenNoData() throws IOException { - TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); - extractor.setNextResponse(createSearchResponse(0L, 0L, 0L)); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1000L, 2300L)); + + DataExtractor summaryExtractor = new StubSubExtractor(new SearchInterval(1000L, 2300L), new DataSummary(null, null, 0L)); + when(dataExtractorFactory.newExtractor(1000L, 2300L)).thenReturn(summaryExtractor); assertThat(extractor.hasNext(), is(true)); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); + + verify(dataExtractorFactory).newExtractor(1000L, 2300L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); } public void testExtractionGivenSpecifiedChunk() throws IOException { chunkSpan = TimeValue.timeValueSeconds(1); - TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); - extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1000L, 2300L)); + + DataExtractor summaryExtractor = new StubSubExtractor(new SearchInterval(1000L, 2300L), new DataSummary(1000L, 2300L, 10L)); + when(dataExtractorFactory.newExtractor(1000L, 2300L)).thenReturn(summaryExtractor); InputStream inputStream1 = mock(InputStream.class); InputStream inputStream2 = mock(InputStream.class); InputStream inputStream3 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(1000L, 2000L), inputStream1, inputStream2); - when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(1000L, 2000L), inputStream1, inputStream2); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtractor1); - DataExtractor subExtactor2 = new StubSubExtractor(new SearchInterval(2000L, 2300L), inputStream3); - when(dataExtractorFactory.newExtractor(2000L, 2300L)).thenReturn(subExtactor2); + DataExtractor subExtractor2 = new StubSubExtractor(new SearchInterval(2000L, 2300L), inputStream3); + when(dataExtractorFactory.newExtractor(2000L, 2300L)).thenReturn(subExtractor2); assertThat(extractor.hasNext(), is(true)); DataExtractor.Result result = extractor.next(); @@ -148,46 +94,31 @@ public void testExtractionGivenSpecifiedChunk() throws IOException { assertThat(result.searchInterval(), equalTo(new SearchInterval(2000L, 2300L))); assertThat(result.data().isPresent(), is(false)); + verify(dataExtractorFactory).newExtractor(1000L, 2300L); verify(dataExtractorFactory).newExtractor(1000L, 2000L); verify(dataExtractorFactory).newExtractor(2000L, 2300L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); - String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); - assertThat(searchRequest, containsString("\"size\":0")); - assertThat( - searchRequest, - containsString( - "\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," - + "{\"range\":{\"time\":{\"gte\":1000,\"lt\":2300," - + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]" - ) - ); - assertThat( - searchRequest, - containsString( - "\"aggregations\":{\"earliest_time\":{\"min\":{\"field\":\"time\"}}," + "\"latest_time\":{\"max\":{\"field\":\"time\"}}}}" - ) - ); - assertThat(searchRequest, not(containsString("\"track_total_hits\":false"))); - assertThat(searchRequest, not(containsString("\"sort\""))); } public void testExtractionGivenSpecifiedChunkAndAggs() throws IOException { chunkSpan = TimeValue.timeValueSeconds(1); - TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L, true, 1000L); - // 0 hits with non-empty data is possible with rollups - extractor.setNextResponse(createSearchResponse(randomFrom(0L, 2L, 10000L), 1000L, 2200L)); + DataExtractor summaryExtractor = new StubSubExtractor( + new SearchInterval(1000L, 2300L), + new DataSummary(1000L, 2200L, randomFrom(0L, 2L, 10000L)) + ); + when(dataExtractorFactory.newExtractor(1000L, 2300L)).thenReturn(summaryExtractor); + + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1000L, 2300L, true, 200L)); InputStream inputStream1 = mock(InputStream.class); InputStream inputStream2 = mock(InputStream.class); InputStream inputStream3 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(1000L, 2000L), inputStream1, inputStream2); - when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(1000L, 2000L), inputStream1, inputStream2); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtractor1); - DataExtractor subExtactor2 = new StubSubExtractor(new SearchInterval(2000L, 2300L), inputStream3); - when(dataExtractorFactory.newExtractor(2000L, 2300L)).thenReturn(subExtactor2); + DataExtractor subExtractor2 = new StubSubExtractor(new SearchInterval(2000L, 2300L), inputStream3); + when(dataExtractorFactory.newExtractor(2000L, 2300L)).thenReturn(subExtractor2); assertThat(extractor.hasNext(), is(true)); DataExtractor.Result result = extractor.next(); @@ -206,47 +137,31 @@ public void testExtractionGivenSpecifiedChunkAndAggs() throws IOException { assertThat(result.searchInterval(), equalTo(new SearchInterval(2000L, 2300L))); assertThat(result.data().isPresent(), is(false)); + verify(dataExtractorFactory).newExtractor(1000L, 2300L); verify(dataExtractorFactory).newExtractor(1000L, 2000L); verify(dataExtractorFactory).newExtractor(2000L, 2300L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); - String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); - assertThat(searchRequest, containsString("\"size\":0")); - assertThat( - searchRequest, - containsString( - "\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," - + "{\"range\":{\"time\":{\"gte\":1000,\"lt\":2300," - + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]" - ) - ); - assertThat( - searchRequest, - containsString( - "\"aggregations\":{\"earliest_time\":{\"min\":{\"field\":\"time\"}}," + "\"latest_time\":{\"max\":{\"field\":\"time\"}}}}" - ) - ); - assertThat(searchRequest, not(containsString("\"track_total_hits\":false"))); - assertThat(searchRequest, not(containsString("\"sort\""))); } public void testExtractionGivenAutoChunkAndAggs() throws IOException { chunkSpan = null; - TestDataExtractor extractor = new TestDataExtractor(100_000L, 450_000L, true, 200L); + DataExtractor summaryExtractor = new StubSubExtractor( + new SearchInterval(100_000L, 450_000L), + new DataSummary(100_000L, 400_000L, randomFrom(0L, 2L, 10000L)) + ); + when(dataExtractorFactory.newExtractor(100_000L, 450_000L)).thenReturn(summaryExtractor); - // 0 hits with non-empty data is possible with rollups - extractor.setNextResponse(createSearchResponse(randomFrom(0L, 2L, 10000L), 100_000L, 400_000L)); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(100_000L, 450_000L, true, 200L)); InputStream inputStream1 = mock(InputStream.class); InputStream inputStream2 = mock(InputStream.class); // 200 * 1_000 == 200_000 - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(100_000L, 300_000L), inputStream1); - when(dataExtractorFactory.newExtractor(100_000L, 300_000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(100_000L, 300_000L), inputStream1); + when(dataExtractorFactory.newExtractor(100_000L, 300_000L)).thenReturn(subExtractor1); - DataExtractor subExtactor2 = new StubSubExtractor(new SearchInterval(300_000L, 450_000L), inputStream2); - when(dataExtractorFactory.newExtractor(300_000L, 450_000L)).thenReturn(subExtactor2); + DataExtractor subExtractor2 = new StubSubExtractor(new SearchInterval(300_000L, 450_000L), inputStream2); + when(dataExtractorFactory.newExtractor(300_000L, 450_000L)).thenReturn(subExtractor2); assertThat(extractor.hasNext(), is(true)); DataExtractor.Result result = extractor.next(); @@ -261,43 +176,47 @@ public void testExtractionGivenAutoChunkAndAggs() throws IOException { assertThat(result.data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); + verify(dataExtractorFactory).newExtractor(100_000L, 450_000L); verify(dataExtractorFactory).newExtractor(100_000L, 300_000L); verify(dataExtractorFactory).newExtractor(300_000L, 450_000L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); } public void testExtractionGivenAutoChunkAndAggsAndNoData() throws IOException { chunkSpan = null; - TestDataExtractor extractor = new TestDataExtractor(100L, 500L, true, 200L); + DataExtractor summaryExtractor = new StubSubExtractor(new SearchInterval(100L, 500L), new DataSummary(null, null, 0L)); + when(dataExtractorFactory.newExtractor(100L, 500L)).thenReturn(summaryExtractor); - extractor.setNextResponse(createNullSearchResponse()); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(100L, 500L, true, 200L)); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); + verify(dataExtractorFactory).newExtractor(100L, 500L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); } public void testExtractionGivenAutoChunkAndScrollSize1000() throws IOException { chunkSpan = null; scrollSize = 1000; - TestDataExtractor extractor = new TestDataExtractor(100000L, 450000L); // 300K millis * 1000 * 10 / 15K docs = 200000 - extractor.setNextResponse(createSearchResponse(15000L, 100000L, 400000L)); + DataExtractor summaryExtractor = new StubSubExtractor( + new SearchInterval(100000L, 450000L), + new DataSummary(100000L, 400000L, 15000L) + ); + when(dataExtractorFactory.newExtractor(100000L, 450000L)).thenReturn(summaryExtractor); + + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(100000L, 450000L)); InputStream inputStream1 = mock(InputStream.class); InputStream inputStream2 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(100_000L, 300_000L), inputStream1); - when(dataExtractorFactory.newExtractor(100_000L, 300_000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(100_000L, 300_000L), inputStream1); + when(dataExtractorFactory.newExtractor(100_000L, 300_000L)).thenReturn(subExtractor1); - DataExtractor subExtactor2 = new StubSubExtractor(new SearchInterval(300_000L, 450_000L), inputStream2); - when(dataExtractorFactory.newExtractor(300_000L, 450_000L)).thenReturn(subExtactor2); + DataExtractor subExtractor2 = new StubSubExtractor(new SearchInterval(300_000L, 450_000L), inputStream2); + when(dataExtractorFactory.newExtractor(300_000L, 450_000L)).thenReturn(subExtractor2); assertThat(extractor.hasNext(), is(true)); assertEquals(inputStream1, extractor.next().data().get()); @@ -306,29 +225,31 @@ public void testExtractionGivenAutoChunkAndScrollSize1000() throws IOException { assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); + verify(dataExtractorFactory).newExtractor(100000L, 450000L); verify(dataExtractorFactory).newExtractor(100000L, 300000L); verify(dataExtractorFactory).newExtractor(300000L, 450000L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); } public void testExtractionGivenAutoChunkAndScrollSize500() throws IOException { chunkSpan = null; scrollSize = 500; - TestDataExtractor extractor = new TestDataExtractor(100000L, 450000L); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(100000L, 450000L)); - // 300K millis * 500 * 10 / 15K docs = 100000 - extractor.setNextResponse(createSearchResponse(15000L, 100000L, 400000L)); + DataExtractor summaryExtractor = new StubSubExtractor( + new SearchInterval(100000L, 450000L), + new DataSummary(100000L, 400000L, 15000L) + ); + when(dataExtractorFactory.newExtractor(100000L, 450000L)).thenReturn(summaryExtractor); InputStream inputStream1 = mock(InputStream.class); InputStream inputStream2 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(100_000L, 200_000L), inputStream1); - when(dataExtractorFactory.newExtractor(100000L, 200000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(100_000L, 200_000L), inputStream1); + when(dataExtractorFactory.newExtractor(100000L, 200000L)).thenReturn(subExtractor1); - DataExtractor subExtactor2 = new StubSubExtractor(new SearchInterval(200_000L, 300_000L), inputStream2); - when(dataExtractorFactory.newExtractor(200000L, 300000L)).thenReturn(subExtactor2); + DataExtractor subExtractor2 = new StubSubExtractor(new SearchInterval(200_000L, 300_000L), inputStream2); + when(dataExtractorFactory.newExtractor(200000L, 300000L)).thenReturn(subExtractor2); assertThat(extractor.hasNext(), is(true)); assertEquals(inputStream1, extractor.next().data().get()); @@ -336,28 +257,31 @@ public void testExtractionGivenAutoChunkAndScrollSize500() throws IOException { assertEquals(inputStream2, extractor.next().data().get()); assertThat(extractor.hasNext(), is(true)); + verify(dataExtractorFactory).newExtractor(100000L, 450000L); verify(dataExtractorFactory).newExtractor(100000L, 200000L); verify(dataExtractorFactory).newExtractor(200000L, 300000L); - - assertThat(capturedSearchRequests.size(), equalTo(1)); } public void testExtractionGivenAutoChunkIsLessThanMinChunk() throws IOException { chunkSpan = null; scrollSize = 1000; - TestDataExtractor extractor = new TestDataExtractor(100000L, 450000L); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(100000L, 450000L)); // 30K millis * 1000 * 10 / 150K docs = 2000 < min of 60K - extractor.setNextResponse(createSearchResponse(150000L, 100000L, 400000L)); + DataExtractor summaryExtractor = new StubSubExtractor( + new SearchInterval(100000L, 450000L), + new DataSummary(100000L, 400000L, 150000L) + ); + when(dataExtractorFactory.newExtractor(100000L, 450000L)).thenReturn(summaryExtractor); InputStream inputStream1 = mock(InputStream.class); InputStream inputStream2 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(100_000L, 160_000L), inputStream1); - when(dataExtractorFactory.newExtractor(100000L, 160000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(100_000L, 160_000L), inputStream1); + when(dataExtractorFactory.newExtractor(100000L, 160000L)).thenReturn(subExtractor1); - DataExtractor subExtactor2 = new StubSubExtractor(new SearchInterval(160_000L, 220_000L), inputStream2); - when(dataExtractorFactory.newExtractor(160000L, 220000L)).thenReturn(subExtactor2); + DataExtractor subExtractor2 = new StubSubExtractor(new SearchInterval(160_000L, 220_000L), inputStream2); + when(dataExtractorFactory.newExtractor(160000L, 220000L)).thenReturn(subExtractor2); assertThat(extractor.hasNext(), is(true)); assertEquals(inputStream1, extractor.next().data().get()); @@ -365,24 +289,24 @@ public void testExtractionGivenAutoChunkIsLessThanMinChunk() throws IOException assertEquals(inputStream2, extractor.next().data().get()); assertThat(extractor.hasNext(), is(true)); + verify(dataExtractorFactory).newExtractor(100000L, 450000L); verify(dataExtractorFactory).newExtractor(100000L, 160000L); verify(dataExtractorFactory).newExtractor(160000L, 220000L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); } public void testExtractionGivenAutoChunkAndDataTimeSpreadIsZero() throws IOException { chunkSpan = null; scrollSize = 1000; - TestDataExtractor extractor = new TestDataExtractor(100L, 500L); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(100L, 500L)); - extractor.setNextResponse(createSearchResponse(150000L, 300L, 300L)); + DataExtractor summaryExtractor = new StubSubExtractor(new SearchInterval(100L, 500L), new DataSummary(300L, 300L, 150000L)); + when(dataExtractorFactory.newExtractor(100L, 500L)).thenReturn(summaryExtractor); InputStream inputStream1 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(300L, 500L), inputStream1); - when(dataExtractorFactory.newExtractor(300L, 500L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(300L, 500L), inputStream1); + when(dataExtractorFactory.newExtractor(300L, 500L)).thenReturn(subExtractor1); assertThat(extractor.hasNext(), is(true)); assertEquals(inputStream1, extractor.next().data().get()); @@ -390,24 +314,20 @@ public void testExtractionGivenAutoChunkAndDataTimeSpreadIsZero() throws IOExcep assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); + verify(dataExtractorFactory).newExtractor(100L, 500L); verify(dataExtractorFactory).newExtractor(300L, 500L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); } public void testExtractionGivenAutoChunkAndTotalTimeRangeSmallerThanChunk() throws IOException { chunkSpan = null; scrollSize = 1000; - TestDataExtractor extractor = new TestDataExtractor(1L, 101L); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1L, 101L)); // 100 millis * 1000 * 10 / 10 docs = 100000 - extractor.setNextResponse(createSearchResponse(10L, 1L, 101L)); - InputStream inputStream1 = mock(InputStream.class); - - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(1L, 10L), inputStream1); - when(dataExtractorFactory.newExtractor(1L, 101L)).thenReturn(subExtactor1); + DataExtractor stubExtractor = new StubSubExtractor(new SearchInterval(1L, 101L), new DataSummary(1L, 101L, 10L), inputStream1); + when(dataExtractorFactory.newExtractor(1L, 101L)).thenReturn(stubExtractor); assertThat(extractor.hasNext(), is(true)); assertEquals(inputStream1, extractor.next().data().get()); @@ -415,19 +335,21 @@ public void testExtractionGivenAutoChunkAndTotalTimeRangeSmallerThanChunk() thro assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); - verify(dataExtractorFactory).newExtractor(1L, 101L); + verify(dataExtractorFactory, times(2)).newExtractor(1L, 101L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(1)); } public void testExtractionGivenAutoChunkAndIntermediateEmptySearchShouldReconfigure() throws IOException { chunkSpan = null; scrollSize = 500; - TestDataExtractor extractor = new TestDataExtractor(100000L, 400000L); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(100000L, 400000L)); // 300K millis * 500 * 10 / 15K docs = 100000 - extractor.setNextResponse(createSearchResponse(15000L, 100000L, 400000L)); + DataExtractor summaryExtractor = new StubSubExtractor( + new SearchInterval(100000L, 400000L), + new DataSummary(100000L, 400000L, 15000L) + ); + when(dataExtractorFactory.newExtractor(100000L, 400000L)).thenReturn(summaryExtractor); InputStream inputStream1 = mock(InputStream.class); @@ -443,44 +365,30 @@ public void testExtractionGivenAutoChunkAndIntermediateEmptySearchShouldReconfig assertThat(extractor.hasNext(), is(true)); // Now we have: 200K millis * 500 * 10 / 5K docs = 200000 - extractor.setNextResponse(createSearchResponse(5000, 200000L, 400000L)); - - // This is the last one InputStream inputStream2 = mock(InputStream.class); - DataExtractor subExtractor3 = new StubSubExtractor(new SearchInterval(200_000L, 400_000L), inputStream2); - when(dataExtractorFactory.newExtractor(200000, 400000)).thenReturn(subExtractor3); + DataExtractor newExtractor = new StubSubExtractor( + new SearchInterval(300000L, 400000L), + new DataSummary(300000L, 400000L, 5000L), + inputStream2 + ); + when(dataExtractorFactory.newExtractor(300000L, 400000L)).thenReturn(newExtractor); assertEquals(inputStream2, extractor.next().data().get()); assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); - verify(dataExtractorFactory).newExtractor(100000L, 200000L); - verify(dataExtractorFactory).newExtractor(200000L, 300000L); - verify(dataExtractorFactory).newExtractor(200000L, 400000L); + verify(dataExtractorFactory).newExtractor(100000L, 400000L); // Initial summary + verify(dataExtractorFactory).newExtractor(100000L, 200000L); // Chunk 1 + verify(dataExtractorFactory).newExtractor(200000L, 300000L); // Chunk 2 with no data + verify(dataExtractorFactory, times(2)).newExtractor(300000L, 400000L); // Reconfigure and new chunk Mockito.verifyNoMoreInteractions(dataExtractorFactory); - - assertThat(capturedSearchRequests.size(), equalTo(2)); - - String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); - assertThat(searchRequest, containsString("\"gte\":100000,\"lt\":400000")); - searchRequest = capturedSearchRequests.get(1).toString().replaceAll("\\s", ""); - assertThat(searchRequest, containsString("\"gte\":300000,\"lt\":400000")); } public void testCancelGivenNextWasNeverCalled() { chunkSpan = TimeValue.timeValueSeconds(1); - TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); - extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); - - InputStream inputStream1 = mock(InputStream.class); - - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(1000L, 2000L), inputStream1); - when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); - + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1000L, 2300L)); assertThat(extractor.hasNext(), is(true)); - extractor.cancel(); - assertThat(extractor.isCancelled(), is(true)); assertThat(extractor.hasNext(), is(false)); Mockito.verifyNoMoreInteractions(dataExtractorFactory); @@ -488,14 +396,16 @@ public void testCancelGivenNextWasNeverCalled() { public void testCancelGivenCurrentSubExtractorHasMore() throws IOException { chunkSpan = TimeValue.timeValueSeconds(1); - TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); - extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1000L, 2300L)); + + DataExtractor summaryExtractor = new StubSubExtractor(new SearchInterval(1000L, 2300L), new DataSummary(1000L, 2200L, 10L)); + when(dataExtractorFactory.newExtractor(1000L, 2300L)).thenReturn(summaryExtractor); InputStream inputStream1 = mock(InputStream.class); InputStream inputStream2 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(1000L, 2000L), inputStream1, inputStream2); - when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(1000L, 2000L), inputStream1, inputStream2); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtractor1); assertThat(extractor.hasNext(), is(true)); assertEquals(inputStream1, extractor.next().data().get()); @@ -509,19 +419,23 @@ public void testCancelGivenCurrentSubExtractorHasMore() throws IOException { assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); + verify(dataExtractorFactory).newExtractor(1000L, 2300L); verify(dataExtractorFactory).newExtractor(1000L, 2000L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); } public void testCancelGivenCurrentSubExtractorIsDone() throws IOException { chunkSpan = TimeValue.timeValueSeconds(1); - TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); - extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); + + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1000L, 2300L)); + + DataExtractor summaryExtractor = new StubSubExtractor(new SearchInterval(1000L, 2300L), new DataSummary(1000L, 2200L, 10L)); + when(dataExtractorFactory.newExtractor(1000L, 2300L)).thenReturn(summaryExtractor); InputStream inputStream1 = mock(InputStream.class); - DataExtractor subExtactor1 = new StubSubExtractor(new SearchInterval(1000L, 3000L), inputStream1); - when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + DataExtractor subExtractor1 = new StubSubExtractor(new SearchInterval(1000L, 3000L), inputStream1); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtractor1); assertThat(extractor.hasNext(), is(true)); assertEquals(inputStream1, extractor.next().data().get()); @@ -533,66 +447,27 @@ public void testCancelGivenCurrentSubExtractorIsDone() throws IOException { assertThat(extractor.next().data().isPresent(), is(false)); assertThat(extractor.hasNext(), is(false)); + verify(dataExtractorFactory).newExtractor(1000L, 2300L); verify(dataExtractorFactory).newExtractor(1000L, 2000L); Mockito.verifyNoMoreInteractions(dataExtractorFactory); } public void testDataSummaryRequestIsFailed() { chunkSpan = TimeValue.timeValueSeconds(2); - TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); - extractor.setNextResponseToError(new SearchPhaseExecutionException("search phase 1", "boom", ShardSearchFailure.EMPTY_ARRAY)); + DataExtractor extractor = new ChunkedDataExtractor(dataExtractorFactory, createContext(1000L, 2300L)); + when(dataExtractorFactory.newExtractor(1000L, 2300L)).thenThrow( + new SearchPhaseExecutionException("search phase 1", "boom", ShardSearchFailure.EMPTY_ARRAY) + ); assertThat(extractor.hasNext(), is(true)); expectThrows(SearchPhaseExecutionException.class, extractor::next); } public void testNoDataSummaryHasNoData() { - ChunkedDataExtractor.DataSummary summary = ChunkedDataExtractor.AggregatedDataSummary.noDataSummary(randomNonNegativeLong()); + DataSummary summary = new DataSummary(null, null, 0L); assertFalse(summary.hasData()); } - private SearchResponse createSearchResponse(long totalHits, long earliestTime, long latestTime) { - SearchResponse searchResponse = mock(SearchResponse.class); - when(searchResponse.status()).thenReturn(RestStatus.OK); - SearchHit[] hits = new SearchHit[(int) totalHits]; - Arrays.fill(hits, SearchHit.unpooled(1)); - SearchHits searchHits = SearchHits.unpooled(hits, new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), 1); - when(searchResponse.getHits()).thenReturn(searchHits); - - List aggs = new ArrayList<>(); - Min min = mock(Min.class); - when(min.value()).thenReturn((double) earliestTime); - when(min.getName()).thenReturn("earliest_time"); - aggs.add(min); - Max max = mock(Max.class); - when(max.value()).thenReturn((double) latestTime); - when(max.getName()).thenReturn("latest_time"); - aggs.add(max); - InternalAggregations aggregations = InternalAggregations.from(aggs); - when(searchResponse.getAggregations()).thenReturn(aggregations); - return searchResponse; - } - - private SearchResponse createNullSearchResponse() { - SearchResponse searchResponse = mock(SearchResponse.class); - when(searchResponse.status()).thenReturn(RestStatus.OK); - SearchHits searchHits = SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), 1); - when(searchResponse.getHits()).thenReturn(searchHits); - - List aggs = new ArrayList<>(); - Min min = mock(Min.class); - when(min.value()).thenReturn(Double.POSITIVE_INFINITY); - when(min.getName()).thenReturn("earliest_time"); - aggs.add(min); - Max max = mock(Max.class); - when(max.value()).thenReturn(Double.POSITIVE_INFINITY); - when(max.getName()).thenReturn("latest_time"); - aggs.add(max); - InternalAggregations aggregations = InternalAggregations.from(aggs); - when(searchResponse.getAggregations()).thenReturn(aggregations); - return searchResponse; - } - private ChunkedDataExtractorContext createContext(long start, long end) { return createContext(start, end, false, null); } @@ -600,32 +475,38 @@ private ChunkedDataExtractorContext createContext(long start, long end) { private ChunkedDataExtractorContext createContext(long start, long end, boolean hasAggregations, Long histogramInterval) { return new ChunkedDataExtractorContext( jobId, - timeField, - indices, - QueryBuilders.matchAllQuery(), scrollSize, start, end, chunkSpan, ChunkedDataExtractorFactory.newIdentityTimeAligner(), - Collections.emptyMap(), hasAggregations, - histogramInterval, - SearchRequest.DEFAULT_INDICES_OPTIONS, - Collections.emptyMap() + histogramInterval ); } private static class StubSubExtractor implements DataExtractor { - final SearchInterval searchInterval; - List streams = new ArrayList<>(); - boolean hasNext = true; + + private final DataSummary summary; + private final SearchInterval searchInterval; + private final List streams = new ArrayList<>(); + private boolean hasNext = true; StubSubExtractor(SearchInterval searchInterval, InputStream... streams) { + this(searchInterval, null, streams); + } + + StubSubExtractor(SearchInterval searchInterval, DataSummary summary, InputStream... streams) { this.searchInterval = searchInterval; + this.summary = summary; Collections.addAll(this.streams, streams); } + @Override + public DataSummary getSummary() { + return summary; + } + @Override public boolean hasNext() { return hasNext; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java index f3eab09b7bc2e..d994b14265a26 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java @@ -31,6 +31,9 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.metrics.Max; +import org.elasticsearch.search.aggregations.metrics.Min; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -39,6 +42,7 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter.DatafeedTimingStatsPersister; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor.DataSummary; import org.elasticsearch.xpack.ml.extractor.DocValueField; import org.elasticsearch.xpack.ml.extractor.ExtractedField; import org.elasticsearch.xpack.ml.extractor.TimeField; @@ -65,6 +69,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -512,6 +517,37 @@ public void testDomainSplitScriptField() throws IOException { assertThat(capturedClearScrollIds.get(0), equalTo(response2.getScrollId())); } + public void testGetSummary() { + ScrollDataExtractorContext context = createContext(1000L, 2300L); + TestDataExtractor extractor = new TestDataExtractor(context); + extractor.setNextResponse(createSummaryResponse(1001L, 2299L, 10L)); + + DataSummary summary = extractor.getSummary(); + assertThat(summary.earliestTime(), equalTo(1001L)); + assertThat(summary.latestTime(), equalTo(2299L)); + assertThat(summary.totalHits(), equalTo(10L)); + + assertThat(capturedSearchRequests.size(), equalTo(1)); + String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + assertThat(searchRequest, containsString("\"size\":0")); + assertThat( + searchRequest, + containsString( + "\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," + + "{\"range\":{\"time\":{\"gte\":1000,\"lt\":2300," + + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]" + ) + ); + assertThat( + searchRequest, + containsString( + "\"aggregations\":{\"earliest_time\":{\"min\":{\"field\":\"time\"}}," + "\"latest_time\":{\"max\":{\"field\":\"time\"}}}}" + ) + ); + assertThat(searchRequest, not(containsString("\"track_total_hits\":false"))); + assertThat(searchRequest, not(containsString("\"sort\""))); + } + private ScrollDataExtractorContext createContext(long start, long end) { return new ScrollDataExtractorContext( jobId, @@ -553,6 +589,17 @@ private SearchResponse createSearchResponse(List timestamps, List return searchResponse; } + private SearchResponse createSummaryResponse(long start, long end, long totalHits) { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn( + new SearchHits(SearchHits.EMPTY, new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), 1) + ); + when(searchResponse.getAggregations()).thenReturn( + InternalAggregations.from(List.of(new Min("earliest_time", start, null, null), new Max("latest_time", end, null, null))) + ); + return searchResponse; + } + private List getCapturedClearScrollIds() { return capturedClearScrollRequests.getAllValues().stream().map(r -> r.getScrollIds().get(0)).collect(Collectors.toList()); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java index 6d963cae8159c..ffa56d3d076f9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManagerTests.java @@ -105,6 +105,7 @@ public void testRejectedExecution() { TimeValue.timeValueMinutes(1), TrainedModelPrefixStrings.PrefixType.NONE, null, + randomBoolean(), ActionListener.wrap(result -> fail("unexpected success"), e -> assertThat(e, instanceOf(EsRejectedExecutionException.class))) ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchActionTests.java index 4fa0876991e3b..f80226b164988 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchActionTests.java @@ -95,6 +95,7 @@ public void testInferListenerOnlyCalledOnce() { TrainedModelPrefixStrings.PrefixType.NONE, tp, null, + randomBoolean(), listener ); action.init(); @@ -117,6 +118,7 @@ public void testInferListenerOnlyCalledOnce() { TrainedModelPrefixStrings.PrefixType.NONE, tp, null, + randomBoolean(), listener ); action.init(); @@ -140,6 +142,7 @@ public void testInferListenerOnlyCalledOnce() { TrainedModelPrefixStrings.PrefixType.NONE, tp, null, + randomBoolean(), listener ); action.init(); @@ -172,6 +175,7 @@ public void testRunNotCalledAfterNotified() { TrainedModelPrefixStrings.PrefixType.NONE, tp, null, + randomBoolean(), listener ); action.init(); @@ -191,6 +195,7 @@ public void testRunNotCalledAfterNotified() { TrainedModelPrefixStrings.PrefixType.NONE, tp, null, + randomBoolean(), listener ); action.init(); @@ -235,6 +240,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, TrainedModelPrefixStrings.PrefixType.NONE, tp, cancellableTask, + randomBoolean(), listener ); action.init(); @@ -261,7 +267,7 @@ public void testPrefixStrings() throws Exception { when(nlpProcessor.getRequestBuilder(any())).thenReturn(requestBuilder); NlpTask.Request builtRequest = new NlpTask.Request(mock(TokenizationResult.class), mock(BytesReference.class)); - when(requestBuilder.buildRequest(anyList(), anyString(), any(), anyInt())).thenReturn(builtRequest); + when(requestBuilder.buildRequest(anyList(), anyString(), any(), anyInt(), anyInt())).thenReturn(builtRequest); when(processContext.getNlpTaskProcessor()).thenReturn(new SetOnce<>(nlpProcessor)); PyTorchResultProcessor resultProcessor = new PyTorchResultProcessor("1", threadSettings -> {}); @@ -286,6 +292,7 @@ public void testPrefixStrings() throws Exception { TrainedModelPrefixStrings.PrefixType.SEARCH, tp, null, + randomBoolean(), listener ); action.init(); @@ -313,6 +320,7 @@ public void testPrefixStrings() throws Exception { TrainedModelPrefixStrings.PrefixType.INGEST, tp, null, + randomBoolean(), listener ); action.init(); @@ -336,6 +344,7 @@ public void testPrefixStrings() throws Exception { TrainedModelPrefixStrings.PrefixType.NONE, tp, null, + randomBoolean(), listener ); action.init(); @@ -363,6 +372,7 @@ public void testPrefixStrings() throws Exception { isForSearch ? TrainedModelPrefixStrings.PrefixType.SEARCH : TrainedModelPrefixStrings.PrefixType.INGEST, tp, null, + randomBoolean(), listener ); action.init(); @@ -394,6 +404,7 @@ public void testPrefixStrings() throws Exception { isForSearch ? TrainedModelPrefixStrings.PrefixType.SEARCH : TrainedModelPrefixStrings.PrefixType.INGEST, tp, null, + randomBoolean(), listener ); action.init(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/BertTokenizationResultTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/BertTokenizationResultTests.java index 6d56bf920fa56..99995b59474c2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/BertTokenizationResultTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/BertTokenizationResultTests.java @@ -41,7 +41,13 @@ public void testBuildRequest() throws IOException { tokenizer = BertTokenizer.builder(TEST_CASED_VOCAB, new BertTokenization(null, null, 512, null, null)).build(); var requestBuilder = tokenizer.requestBuilder(); - NlpTask.Request request = requestBuilder.buildRequest(List.of("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1); + NlpTask.Request request = requestBuilder.buildRequest( + List.of("Elasticsearch fun"), + "request1", + Tokenization.Truncate.NONE, + -1, + null + ); Map jsonDocAsMap = XContentHelper.convertToMap(request.processInput(), true, XContentType.JSON).v2(); assertThat(jsonDocAsMap.keySet(), hasSize(5)); @@ -71,7 +77,8 @@ public void testInputTooLarge() throws IOException { Collections.singletonList("Elasticsearch fun Elasticsearch fun Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, - -1 + -1, + null ) ); @@ -84,7 +91,7 @@ public void testInputTooLarge() throws IOException { var requestBuilder = tokenizer.requestBuilder(); // input will become 3 tokens + the Class and Separator token = 5 which is // our max sequence length - requestBuilder.buildRequest(Collections.singletonList("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1); + requestBuilder.buildRequest(Collections.singletonList("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1, null); } } @@ -97,7 +104,8 @@ public void testBatchWithPadding() throws IOException { List.of("Elasticsearch", "my little red car", "Godzilla day"), "request1", Tokenization.Truncate.NONE, - -1 + -1, + null ); Map jsonDocAsMap = XContentHelper.convertToMap(request.processInput(), true, XContentType.JSON).v2(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessorTests.java index f3afb0286f076..c4bff72f5b25e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/FillMaskProcessorTests.java @@ -67,7 +67,8 @@ public void testProcessResults() { new PyTorchInferenceResult(scores), tokenizer, 4, - resultsField + resultsField, + false ); assertThat(result.asMap().get(resultsField), equalTo("France")); assertThat(result.getTopClasses(), hasSize(4)); @@ -94,7 +95,7 @@ public void testProcessResults_GivenMissingTokens() { PyTorchInferenceResult pyTorchResult = new PyTorchInferenceResult(new double[][][] { { {} } }); expectThrows( ElasticsearchStatusException.class, - () -> FillMaskProcessor.processResult(tokenization, pyTorchResult, tokenizer, 5, randomAlphaOfLength(10)) + () -> FillMaskProcessor.processResult(tokenization, pyTorchResult, tokenizer, 5, randomAlphaOfLength(10), false) ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/MPNetTokenizationResultTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/MPNetTokenizationResultTests.java index fdf9d3c57eb16..86b29baccafd8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/MPNetTokenizationResultTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/MPNetTokenizationResultTests.java @@ -40,7 +40,13 @@ public void testBuildRequest() throws IOException { tokenizer = MPNetTokenizer.mpBuilder(TEST_CASED_VOCAB, new MPNetTokenization(null, null, 512, null, null)).build(); var requestBuilder = tokenizer.requestBuilder(); - NlpTask.Request request = requestBuilder.buildRequest(List.of("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1); + NlpTask.Request request = requestBuilder.buildRequest( + List.of("Elasticsearch fun"), + "request1", + Tokenization.Truncate.NONE, + -1, + null + ); Map jsonDocAsMap = XContentHelper.convertToMap(request.processInput(), true, XContentType.JSON).v2(); assertThat(jsonDocAsMap.keySet(), hasSize(3)); @@ -68,7 +74,8 @@ public void testInputTooLarge() throws IOException { Collections.singletonList("Elasticsearch fun Elasticsearch fun Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, - -1 + -1, + null ) ); @@ -81,7 +88,7 @@ public void testInputTooLarge() throws IOException { var requestBuilder = tokenizer.requestBuilder(); // input will become 3 tokens + the Class and Separator token = 5 which is // our max sequence length - requestBuilder.buildRequest(Collections.singletonList("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1); + requestBuilder.buildRequest(Collections.singletonList("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1, null); } } @@ -94,7 +101,8 @@ public void testBatchWithPadding() throws IOException { List.of("Elasticsearch", "my little red car", "Godzilla day"), "request1", Tokenization.Truncate.NONE, - -1 + -1, + null ); Map jsonDocAsMap = XContentHelper.convertToMap(request.processInput(), true, XContentType.JSON).v2(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessorTests.java index 389a4fab802a0..e8ce230a4acdc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessorTests.java @@ -72,7 +72,7 @@ public void testProcessResults_GivenNoTokens() { var e = expectThrows( ElasticsearchStatusException.class, - () -> processor.processResult(tokenization, new PyTorchInferenceResult(null)) + () -> processor.processResult(tokenization, new PyTorchInferenceResult(null), false) ); assertThat(e, instanceOf(ElasticsearchStatusException.class)); } @@ -98,7 +98,7 @@ public void testProcessResultsWithSpecialTokens() { ).build() ) { TokenizationResult tokenization = tokenizer.buildTokenizationResult( - List.of(tokenizer.tokenize("Many use Elasticsearch in London", Tokenization.Truncate.NONE, -1, 1).get(0)) + List.of(tokenizer.tokenize("Many use Elasticsearch in London", Tokenization.Truncate.NONE, -1, 1, null).get(0)) ); double[][][] scores = { @@ -113,7 +113,7 @@ public void testProcessResultsWithSpecialTokens() { { 0, 0, 0, 0, 0, 0, 0, 6, 0 }, // london { 7, 0, 0, 0, 0, 0, 0, 0, 0 } // sep } }; - NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores)); + NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores), false); assertThat(result.getAnnotatedResult(), equalTo("Many use [Elasticsearch](ORG&Elasticsearch) in [London](LOC&London)")); assertThat(result.getEntityGroups().size(), equalTo(2)); @@ -141,7 +141,7 @@ public void testProcessResults() { { 0, 0, 0, 0, 0, 0, 0, 0, 0 }, // in { 0, 0, 0, 0, 0, 0, 0, 6, 0 } // london } }; - NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores)); + NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores), false); assertThat(result.getAnnotatedResult(), equalTo("Many use [Elasticsearch](ORG&Elasticsearch) in [London](LOC&London)")); assertThat(result.getEntityGroups().size(), equalTo(2)); @@ -178,7 +178,7 @@ public void testProcessResults_withIobMap() { { 0, 0, 0, 0, 0, 0, 0, 0, 5 }, // in { 6, 0, 0, 0, 0, 0, 0, 0, 0 } // london } }; - NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores)); + NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores), false); assertThat(result.getAnnotatedResult(), equalTo("[Elasticsearch](ORG&Elasticsearch) in [London](LOC&London)")); assertThat(result.getEntityGroups().size(), equalTo(2)); @@ -211,7 +211,7 @@ public void testProcessResults_withCustomIobMap() { { 0, 0, 0, 0, 5 }, // in { 6, 0, 0, 0, 0 } // london } }; - NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores)); + NerResults result = (NerResults) processor.processResult(tokenization, new PyTorchInferenceResult(scores), false); assertThat(result.getAnnotatedResult(), equalTo("[Elasticsearch](SOFTWARE&Elasticsearch) in [London](LOC&London)")); assertThat(result.getEntityGroups().size(), equalTo(2)); @@ -370,7 +370,7 @@ private static TokenizationResult tokenize(List vocab, String input) { .setWithSpecialTokens(false) .build() ) { - return tokenizer.buildTokenizationResult(tokenizer.tokenize(input, Tokenization.Truncate.NONE, -1, 0)); + return tokenizer.buildTokenizationResult(tokenizer.tokenize(input, Tokenization.Truncate.NONE, -1, 0, null)); } } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessorTests.java index 48d8c154ae59e..7c2bde5093990 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/QuestionAnsweringProcessorTests.java @@ -80,7 +80,7 @@ public void testProcessor() throws IOException { QuestionAnsweringConfig config = new QuestionAnsweringConfig(question, 1, 10, new VocabularyConfig(""), tokenization, "prediction"); QuestionAnsweringProcessor processor = new QuestionAnsweringProcessor(tokenizer); TokenizationResult tokenizationResult = processor.getRequestBuilder(config) - .buildRequest(List.of(input), "1", Tokenization.Truncate.NONE, 128) + .buildRequest(List.of(input), "1", Tokenization.Truncate.NONE, 128, null) .tokenization(); assertThat(tokenizationResult.anyTruncated(), is(false)); assertThat(tokenizationResult.getTokenization(0).tokenIds().length, equalTo(END_TOKEN_SCORES.length)); @@ -91,7 +91,8 @@ public void testProcessor() throws IOException { PyTorchInferenceResult pyTorchResult = new PyTorchInferenceResult(scores); QuestionAnsweringInferenceResults result = (QuestionAnsweringInferenceResults) resultProcessor.processResult( tokenizationResult, - pyTorchResult + pyTorchResult, + false ); // Note this is a different answer to testTopScores because of the question length @@ -188,7 +189,7 @@ public void testProcessorMuliptleSpans() throws IOException { ); QuestionAnsweringProcessor processor = new QuestionAnsweringProcessor(tokenizer); TokenizationResult tokenizationResult = processor.getRequestBuilder(config) - .buildRequest(List.of(input), "1", Tokenization.Truncate.NONE, span) + .buildRequest(List.of(input), "1", Tokenization.Truncate.NONE, span, null) .tokenization(); assertThat(tokenizationResult.anyTruncated(), is(false)); @@ -223,7 +224,8 @@ public void testProcessorMuliptleSpans() throws IOException { PyTorchInferenceResult pyTorchResult = new PyTorchInferenceResult(modelTensorOutput); QuestionAnsweringInferenceResults result = (QuestionAnsweringInferenceResults) resultProcessor.processResult( tokenizationResult, - pyTorchResult + pyTorchResult, + false ); // The expected answer is the full text of the span containing the answer diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessorTests.java index 1ccd14a7e93c1..c1b2ea591ec15 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextClassificationProcessorTests.java @@ -35,7 +35,14 @@ public void testInvalidResult() { PyTorchInferenceResult torchResult = new PyTorchInferenceResult(new double[][][] {}); var e = expectThrows( ElasticsearchStatusException.class, - () -> TextClassificationProcessor.processResult(null, torchResult, randomInt(), List.of("a", "b"), randomAlphaOfLength(10)) + () -> TextClassificationProcessor.processResult( + null, + torchResult, + randomInt(), + List.of("a", "b"), + randomAlphaOfLength(10), + false + ) ); assertThat(e, instanceOf(ElasticsearchStatusException.class)); assertThat(e.getMessage(), containsString("Text classification result has no data")); @@ -44,7 +51,14 @@ public void testInvalidResult() { PyTorchInferenceResult torchResult = new PyTorchInferenceResult(new double[][][] { { { 1.0 } } }); var e = expectThrows( ElasticsearchStatusException.class, - () -> TextClassificationProcessor.processResult(null, torchResult, randomInt(), List.of("a", "b"), randomAlphaOfLength(10)) + () -> TextClassificationProcessor.processResult( + null, + torchResult, + randomInt(), + List.of("a", "b"), + randomAlphaOfLength(10), + false + ) ); assertThat(e, instanceOf(ElasticsearchStatusException.class)); assertThat(e.getMessage(), containsString("Expected exactly [2] values in text classification result; got [1]")); @@ -69,7 +83,7 @@ public void testBuildRequest() throws IOException { TextClassificationProcessor processor = new TextClassificationProcessor(tokenizer, config); NlpTask.Request request = processor.getRequestBuilder(config) - .buildRequest(List.of("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1); + .buildRequest(List.of("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1, null); Map jsonDocAsMap = XContentHelper.convertToMap(request.processInput(), true, XContentType.JSON).v2(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextEmbeddingProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextEmbeddingProcessorTests.java new file mode 100644 index 0000000000000..ba93feee5c42c --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextEmbeddingProcessorTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.inference.nlp; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertTokenization; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.Tokenization; +import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.BertTokenizationResult; +import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.BertTokenizer; +import org.elasticsearch.xpack.ml.inference.pytorch.results.PyTorchInferenceResult; + +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; + +public class TextEmbeddingProcessorTests extends ESTestCase { + + public void testSingleResult() { + try ( + BertTokenizer tokenizer = BertTokenizer.builder( + TextExpansionProcessorTests.TEST_CASED_VOCAB, + new BertTokenization(null, false, 5, Tokenization.Truncate.NONE, 0) + ).build() + ) { + var pytorchResult = new PyTorchInferenceResult(new double[][][] { { { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 } } }); + + var input = "Elasticsearch darts champion"; + var tokenization = tokenizer.tokenize(input, Tokenization.Truncate.NONE, 0, 0, null); + var tokenizationResult = new BertTokenizationResult(TextExpansionProcessorTests.TEST_CASED_VOCAB, tokenization, 0); + var inferenceResult = TextEmbeddingProcessor.processResult(tokenizationResult, pytorchResult, "foo", false); + assertThat(inferenceResult, instanceOf(TextEmbeddingResults.class)); + + var result = (TextEmbeddingResults) inferenceResult; + assertThat(result.getInference().length, greaterThan(0)); + } + } + + public void testChunking() { + try ( + BertTokenizer tokenizer = BertTokenizer.builder( + TextExpansionProcessorTests.TEST_CASED_VOCAB, + new BertTokenization(null, false, 5, Tokenization.Truncate.NONE, 0) + ).build() + ) { + var pytorchResult = new PyTorchInferenceResult( + new double[][][] { { { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 }, { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 } } } + ); + + var input = "Elasticsearch darts champion little red is fun car"; + var tokenization = tokenizer.tokenize(input, Tokenization.Truncate.NONE, 0, 0, null); + var tokenizationResult = new BertTokenizationResult(TextExpansionProcessorTests.TEST_CASED_VOCAB, tokenization, 0); + var inferenceResult = TextEmbeddingProcessor.processResult(tokenizationResult, pytorchResult, "foo", true); + assertThat(inferenceResult, instanceOf(ChunkedTextEmbeddingResults.class)); + + var chunkedResult = (ChunkedTextEmbeddingResults) inferenceResult; + assertThat(chunkedResult.getChunks(), hasSize(2)); + assertEquals("Elasticsearch darts champion little red", chunkedResult.getChunks().get(0).matchedText()); + assertEquals("is fun car", chunkedResult.getChunks().get(1).matchedText()); + assertThat(chunkedResult.getChunks().get(0).embedding().length, greaterThan(0)); + assertThat(chunkedResult.getChunks().get(1).embedding().length, greaterThan(0)); + } + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java index c94775b1785c9..2af4e599631dc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ml.inference.nlp; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfig; @@ -20,11 +21,30 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.core.IsNot.not; public class TextExpansionProcessorTests extends ESTestCase { + public static final List TEST_CASED_VOCAB = List.of( + BertTokenizer.PAD_TOKEN, + "Elasticsearch", + "is", + "fun", + "my", + "little", + "red", + "car", + "darts", + "champion", + BertTokenizer.CLASS_TOKEN, + BertTokenizer.SEPARATOR_TOKEN, + BertTokenizer.MASK_TOKEN, + BertTokenizer.UNKNOWN_TOKEN + ); + public void testProcessResult() { double[][][] pytorchResult = new double[][][] { { { 0.0, 1.0, 0.0, 3.0, 4.0, 0.0, 0.0 } } }; @@ -34,7 +54,8 @@ public void testProcessResult() { tokenizationResult, new PyTorchInferenceResult(pytorchResult), Map.of(), - "foo" + "foo", + false ); assertThat(inferenceResult, instanceOf(TextExpansionResults.class)); var results = (TextExpansionResults) inferenceResult; @@ -47,30 +68,6 @@ public void testProcessResult() { assertEquals(new TextExpansionResults.WeightedToken("b", 1.0f), weightedTokens.get(2)); } - public void testProcessResultMultipleVectors() { - double[][][] pytorchResult = new double[][][] { { { 0.0, 1.0, 0.0, 1.0, 4.0, 0.0, 0.0 }, { 1.0, 2.0, 0.0, 3.0, 4.0, 0.0, 0.1 } } }; - - TokenizationResult tokenizationResult = new BertTokenizationResult(List.of("a", "b", "c", "d", "e", "f", "g"), List.of(), 0); - - var inferenceResult = TextExpansionProcessor.processResult( - tokenizationResult, - new PyTorchInferenceResult(pytorchResult), - Map.of(), - "foo" - ); - assertThat(inferenceResult, instanceOf(TextExpansionResults.class)); - var results = (TextExpansionResults) inferenceResult; - assertEquals(results.getResultsField(), "foo"); - - var weightedTokens = results.getWeightedTokens(); - assertThat(weightedTokens, hasSize(5)); - assertEquals(new TextExpansionResults.WeightedToken("e", 4.0f), weightedTokens.get(0)); - assertEquals(new TextExpansionResults.WeightedToken("d", 3.0f), weightedTokens.get(1)); - assertEquals(new TextExpansionResults.WeightedToken("b", 2.0f), weightedTokens.get(2)); - assertEquals(new TextExpansionResults.WeightedToken("a", 1.0f), weightedTokens.get(3)); - assertEquals(new TextExpansionResults.WeightedToken("g", 0.1f), weightedTokens.get(4)); - } - public void testSanitiseVocab() { double[][][] pytorchResult = new double[][][] { { { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 } } }; @@ -80,7 +77,8 @@ public void testSanitiseVocab() { tokenizationResult, new PyTorchInferenceResult(pytorchResult), Map.of(4, "XXX", 3, "YYY"), - "foo" + "foo", + false ); assertThat(inferenceResult, instanceOf(TextExpansionResults.class)); var results = (TextExpansionResults) inferenceResult; @@ -113,7 +111,7 @@ public void testSanitizeOutputTokens() { var pytorchResult = new PyTorchInferenceResult(new double[][][] { { { 1.0, 2.0, 3.0, 4.0, 5.0 } } }); TokenizationResult tokenizationResult = new BertTokenizationResult(vocab, List.of(), 0); - TextExpansionResults results = (TextExpansionResults) resultProcessor.processResult(tokenizationResult, pytorchResult); + TextExpansionResults results = (TextExpansionResults) resultProcessor.processResult(tokenizationResult, pytorchResult, false); var weightedTokens = results.getWeightedTokens(); assertThat(weightedTokens, hasSize(5)); assertEquals(new TextExpansionResults.WeightedToken("##__", 5.0f), weightedTokens.get(0)); @@ -122,4 +120,30 @@ public void testSanitizeOutputTokens() { assertEquals(new TextExpansionResults.WeightedToken("bb", 2.0f), weightedTokens.get(3)); assertEquals(new TextExpansionResults.WeightedToken("aa", 1.0f), weightedTokens.get(4)); } + + public void testChunking() { + try ( + BertTokenizer tokenizer = BertTokenizer.builder( + TEST_CASED_VOCAB, + new BertTokenization(null, false, 5, Tokenization.Truncate.NONE, 0) + ).build() + ) { + var pytorchResult = new PyTorchInferenceResult( + new double[][][] { { { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 }, { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 } } } + ); + + var input = "Elasticsearch darts champion little red is fun car"; + var tokenization = tokenizer.tokenize(input, Tokenization.Truncate.NONE, 0, 0, null); + var tokenizationResult = new BertTokenizationResult(TEST_CASED_VOCAB, tokenization, 0); + var inferenceResult = TextExpansionProcessor.processResult(tokenizationResult, pytorchResult, Map.of(), "foo", true); + assertThat(inferenceResult, instanceOf(ChunkedTextExpansionResults.class)); + + var chunkedResult = (ChunkedTextExpansionResults) inferenceResult; + assertThat(chunkedResult.getChunks(), hasSize(2)); + assertEquals("Elasticsearch darts champion little red", chunkedResult.getChunks().get(0).matchedText()); + assertEquals("is fun car", chunkedResult.getChunks().get(1).matchedText()); + assertThat(chunkedResult.getChunks().get(0).weightedTokens(), not(empty())); + assertThat(chunkedResult.getChunks().get(1).weightedTokens(), not(empty())); + } + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessorTests.java index 10be6225163b6..3590793b81abd 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextSimilarityProcessorTests.java @@ -43,7 +43,7 @@ public void testProcessor() throws IOException { ); TextSimilarityProcessor processor = new TextSimilarityProcessor(tokenizer); TokenizationResult tokenizationResult = processor.getRequestBuilder(textSimilarityConfig) - .buildRequest(List.of(input), "1", Tokenization.Truncate.NONE, 128) + .buildRequest(List.of(input), "1", Tokenization.Truncate.NONE, 128, null) .tokenization(); assertThat(tokenizationResult.anyTruncated(), is(false)); assertThat(tokenizationResult.getTokenization(0).tokenIds().length, equalTo(19)); @@ -54,7 +54,8 @@ public void testProcessor() throws IOException { PyTorchInferenceResult pyTorchResult = new PyTorchInferenceResult(scores); TextSimilarityInferenceResults result = (TextSimilarityInferenceResults) resultProcessor.processResult( tokenizationResult, - pyTorchResult + pyTorchResult, + false ); // Note this is a different answer to testTopScores because of the question length @@ -77,7 +78,8 @@ public void testResultFunctions() { PyTorchInferenceResult pyTorchResult = new PyTorchInferenceResult(scores); TextSimilarityInferenceResults result = (TextSimilarityInferenceResults) resultProcessor.processResult( new BertTokenizationResult(List.of(), List.of(), 1), - pyTorchResult + pyTorchResult, + false ); assertThat(result.predictedValue(), equalTo(100.0)); // Test mean @@ -92,7 +94,8 @@ public void testResultFunctions() { resultProcessor = processor.getResultProcessor(textSimilarityConfig); result = (TextSimilarityInferenceResults) resultProcessor.processResult( new BertTokenizationResult(List.of(), List.of(), 1), - pyTorchResult + pyTorchResult, + false ); assertThat(result.predictedValue(), closeTo(51.333333333333, 1e-12)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessorTests.java index d8f1a1fd7433d..6e5274db12593 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/ZeroShotClassificationProcessorTests.java @@ -48,7 +48,7 @@ public void testBuildRequest() throws IOException { NlpTask.Request request = processor.getRequestBuilder( (NlpConfig) config.apply(new ZeroShotClassificationConfigUpdate.Builder().setLabels(List.of("new", "stuff")).build()) - ).buildRequest(List.of("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1); + ).buildRequest(List.of("Elasticsearch fun"), "request1", Tokenization.Truncate.NONE, -1, null); Map jsonDocAsMap = XContentHelper.convertToMap(request.processInput(), true, XContentType.JSON).v2(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertJapaneseTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertJapaneseTokenizerTests.java index 59c7b4df0146c..b3226c72954f3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertJapaneseTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertJapaneseTokenizerTests.java @@ -54,7 +54,7 @@ public void testTokenize() { ) { String msg = "日本語で、ElasticsearchのBertJapaneseTokenizerを使うテスト。"; - TokenizationResult.Tokens tokenization = tokenizer.tokenize(msg, Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize(msg, Tokenization.Truncate.NONE, -1, 0, null).get(0); assertThat( tokenStrings(tokenization.tokens().get(0)), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java index f2c1226f0a04a..901fea45d9de9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java @@ -20,6 +20,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.Tokenization.DEFAULT_MAX_SEQUENCE_LENGTH; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -63,7 +64,13 @@ public void testTokenize() { new BertTokenization(null, false, null, Tokenization.Truncate.NONE, -1) ).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize( + "Elasticsearch fun", + Tokenization.Truncate.NONE, + -1, + 0, + DEFAULT_MAX_SEQUENCE_LENGTH + ).get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("Elastic", "##search", "fun")); assertArrayEquals(new int[] { 0, 1, 3 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 0, 1 }, tokenization.tokenMap()); @@ -126,10 +133,11 @@ public void testTokenizeWithTokensThatAreRemovedByStripAccents() { BertTokenizer tokenizer = BertTokenizer.builder(vocab, new BertTokenization(true, true, null, Tokenization.Truncate.NONE, -1)) .build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize(inputWithAccentsToStrip1, Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize(inputWithAccentsToStrip1, Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenization.tokenIds(), equalTo(new int[] { 37, 0, 1, 3, 2, 4, 12, 11, 10, 9, 8, 7, 10, 5, 13, 14, 38 })); - tokenization = tokenizer.tokenize(inputWithAccentsToStrip2, Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize(inputWithAccentsToStrip2, Tokenization.Truncate.NONE, -1, 0, null).get(0); assertThat( tokenization.tokenIds(), equalTo( @@ -169,13 +177,13 @@ public void testTokenizeWithTokensThatAreRemovedByStripAccents() { ) ); - tokenization = tokenizer.tokenize(inputWithAccentToStripAtEndOfString, Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize(inputWithAccentToStripAtEndOfString, Tokenization.Truncate.NONE, -1, 0, null).get(0); assertThat(tokenization.tokenIds(), equalTo(new int[] { 37, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 38 })); // the last token is the separator, the one before that should // correspond to the last word in the input _not_ the accent assertEquals("cancer", vocab.get(26)); - tokenization = tokenizer.tokenize(onlyAccents, Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize(onlyAccents, Tokenization.Truncate.NONE, -1, 0, null).get(0); // empty tokenization only contains ClASS and SEP tokens assertThat(tokenization.tokenIds(), equalTo(new int[] { 37, 38 })); } @@ -202,10 +210,10 @@ public void testTokenizeFailureCaseAccentFilter() { new BertTokenization(true, true, 512, Tokenization.Truncate.FIRST, -1) ).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Br창n's", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Br창n's", Tokenization.Truncate.NONE, -1, 0, null).get(0); assertThat(tokenization.tokenIds(), equalTo(new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 })); - tokenization = tokenizer.tokenize("Br창n", Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize("Br창n", Tokenization.Truncate.NONE, -1, 0, null).get(0); assertThat(tokenization.tokenIds(), equalTo(new int[] { 0, 1, 2, 3, 4, 5, 8 })); } } @@ -224,17 +232,17 @@ public void testTokenizeLargeInputNoTruncation() { ElasticsearchStatusException ex = expectThrows( ElasticsearchStatusException.class, - () -> tokenizer.tokenize("Elasticsearch fun with Pancake and Godzilla", Tokenization.Truncate.NONE, -1, 0) + () -> tokenizer.tokenize("Elasticsearch fun with Pancake and Godzilla", Tokenization.Truncate.NONE, -1, 0, null) ); assertThat(ex.getMessage(), equalTo("Input too large. The tokenized input length [8] exceeds the maximum sequence length [5]")); // Shouldn't throw - tokenizer.tokenize("Elasticsearch fun with Pancake", Tokenization.Truncate.NONE, -1, 0); + tokenizer.tokenize("Elasticsearch fun with Pancake", Tokenization.Truncate.NONE, -1, 0, null); // Should throw as special chars add two tokens expectThrows( ElasticsearchStatusException.class, - () -> specialCharTokenizer.tokenize("Elasticsearch fun with Pancake", Tokenization.Truncate.NONE, -1, 0) + () -> specialCharTokenizer.tokenize("Elasticsearch fun with Pancake", Tokenization.Truncate.NONE, -1, 0, null) ); } } @@ -250,7 +258,8 @@ public void testTokenizeLargeInputNoTruncationWithWindowing() { "Pancake day fun with Elasticsearch and Godzilla", Tokenization.Truncate.NONE, 0, - 0 + 0, + null ); assertThat(tokens.size(), equalTo(3)); assertArrayEquals(new int[] { 12, 17, 16, 3, 13 }, tokens.get(0).tokenIds()); @@ -262,7 +271,13 @@ public void testTokenizeLargeInputNoTruncationWithWindowing() { assertArrayEquals(new int[] { 12, 15, 8, 9, 13 }, tokens.get(2).tokenIds()); assertArrayEquals(new int[] { -1, 5, 6, 6, -1 }, tokens.get(2).tokenMap()); - tokens = tokenizer.tokenize("Godzilla Elasticsearch day with Pancake and my little red car.", Tokenization.Truncate.NONE, 0, 0); + tokens = tokenizer.tokenize( + "Godzilla Elasticsearch day with Pancake and my little red car.", + Tokenization.Truncate.NONE, + 0, + 0, + null + ); assertThat(tokens.size(), equalTo(5)); assertArrayEquals(new int[] { 12, 8, 9, 13 }, tokens.get(0).tokenIds()); assertArrayEquals(new int[] { -1, 0, 0, -1 }, tokens.get(0).tokenMap()); @@ -283,7 +298,13 @@ public void testTokenizeLargeInputNoTruncationWithWindowing() { // is such that we cannot // In this case, with a span of one and two split words next to eachother, the first subsequence has // "God ##zilla" and the next starts "##zilla Elastic ##search" - tokens = tokenizer.tokenize("Godzilla Elasticsearch day with Pancake and my little red car.", Tokenization.Truncate.NONE, 1, 0); + tokens = tokenizer.tokenize( + "Godzilla Elasticsearch day with Pancake and my little red car.", + Tokenization.Truncate.NONE, + 1, + 0, + null + ); assertThat(tokens.size(), equalTo(7)); assertArrayEquals(new int[] { 12, 8, 9, 13 }, tokens.get(0).tokenIds()); assertArrayEquals(new int[] { -1, 0, 0, -1 }, tokens.get(0).tokenMap()); @@ -300,7 +321,8 @@ public void testTokenizeLargeInputNoTruncationWithWindowing() { "Godzilla Elasticsearch day with Pancake and my little red car.", Tokenization.Truncate.NONE, 1, - 0 + 0, + null ); assertThat(tokens.size(), equalTo(3)); assertArrayEquals(new int[] { 8, 9, 0, 1, 16 }, tokens.get(0).tokenIds()); @@ -326,7 +348,8 @@ public void testTokenizeLargeInputTruncation() { "Elasticsearch fun with Pancake and Godzilla", Tokenization.Truncate.FIRST, -1, - 0 + 0, + null ).get(0); assertArrayEquals(new int[] { 0, 1, 3, 18, 17 }, tokenization.tokenIds()); } @@ -341,7 +364,8 @@ public void testTokenizeLargeInputTruncation() { "Elasticsearch fun with Pancake and Godzilla", Tokenization.Truncate.FIRST, -1, - 0 + 0, + null ).get(0); assertArrayEquals(new int[] { 12, 0, 1, 3, 13 }, tokenization.tokenIds()); assertArrayEquals(new int[] { -1, 0, 0, 1, -1 }, tokenization.tokenMap()); @@ -350,7 +374,8 @@ public void testTokenizeLargeInputTruncation() { public void testTokenizeAppendSpecialTokens() { try (BertTokenizer tokenizer = BertTokenizer.builder(TEST_CASED_VOCAB, Tokenization.createDefault()).build()) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertArrayEquals(new int[] { 12, 0, 1, 3, 13 }, tokenization.tokenIds()); assertArrayEquals(new int[] { -1, 0, 0, 1, -1 }, tokenization.tokenMap()); } @@ -370,7 +395,8 @@ public void testNeverSplitTokens() { "Elasticsearch " + specialToken + " fun", Tokenization.Truncate.NONE, -1, - 0 + 0, + null ).get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("Elastic", "##search", specialToken, "fun")); assertArrayEquals(new int[] { 0, 1, 15, 3 }, tokenization.tokenIds()); @@ -386,11 +412,12 @@ public void testDoLowerCase() { ).setDoLowerCase(false).setWithSpecialTokens(false).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertArrayEquals(new int[] { 3, 2 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 1 }, tokenization.tokenMap()); - tokenization = tokenizer.tokenize("elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize("elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null).get(0); assertArrayEquals(new int[] { 0, 1, 2 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 0, 1 }, tokenization.tokenMap()); } @@ -402,7 +429,8 @@ public void testDoLowerCase() { ).setDoLowerCase(true).setWithSpecialTokens(false).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertArrayEquals(new int[] { 0, 1, 2 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 0, 1 }, tokenization.tokenMap()); } @@ -414,12 +442,13 @@ public void testPunctuation() { .setWithSpecialTokens(false) .build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch, fun.", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch, fun.", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("Elastic", "##search", ",", "fun", ".")); assertArrayEquals(new int[] { 0, 1, 11, 3, 10 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 0, 1, 2, 3 }, tokenization.tokenMap()); - tokenization = tokenizer.tokenize("Elasticsearch, fun [MASK].", Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize("Elasticsearch, fun [MASK].", Tokenization.Truncate.NONE, -1, 0, null).get(0); assertArrayEquals(new int[] { 0, 1, 11, 3, 14, 10 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 0, 1, 2, 3, 4 }, tokenization.tokenMap()); } @@ -449,17 +478,18 @@ public void testPunctuationWithMask() { ).setWithSpecialTokens(true).setNeverSplit(Set.of("[MASK]")).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("This is [MASK]-tastic!", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("This is [MASK]-tastic!", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("This", "is", "[MASK]", "-", "ta", "##stic", "!")); assertArrayEquals(new int[] { 0, 1, 2, 3, 4, 6, 7, 8, 9 }, tokenization.tokenIds()); assertArrayEquals(new int[] { -1, 0, 1, 2, 3, 4, 4, 5, -1 }, tokenization.tokenMap()); - tokenization = tokenizer.tokenize("This is sub~[MASK]!", Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize("This is sub~[MASK]!", Tokenization.Truncate.NONE, -1, 0, null).get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("This", "is", "sub", "~", "[MASK]", "!")); assertArrayEquals(new int[] { 0, 1, 2, 10, 5, 3, 8, 9 }, tokenization.tokenIds()); assertArrayEquals(new int[] { -1, 0, 1, 2, 3, 4, 5, -1 }, tokenization.tokenMap()); - tokenization = tokenizer.tokenize("This is sub,[MASK].tastic!", Tokenization.Truncate.NONE, -1, 0).get(0); + tokenization = tokenizer.tokenize("This is sub,[MASK].tastic!", Tokenization.Truncate.NONE, -1, 0, null).get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("This", "is", "sub", ",", "[MASK]", ".", "ta", "##stic", "!")); assertArrayEquals(new int[] { 0, 1, 2, 10, 11, 3, 12, 6, 7, 8, 9 }, tokenization.tokenIds()); assertArrayEquals(new int[] { -1, 0, 1, 2, 3, 4, 5, 6, 6, 7, -1 }, tokenization.tokenMap()); @@ -476,10 +506,10 @@ public void testBatchInput() { TokenizationResult tr = tokenizer.buildTokenizationResult( List.of( - tokenizer.tokenize("Elasticsearch", Tokenization.Truncate.NONE, -1, 0).get(0), - tokenizer.tokenize("my little red car", Tokenization.Truncate.NONE, -1, 1).get(0), - tokenizer.tokenize("Godzilla day", Tokenization.Truncate.NONE, -1, 2).get(0), - tokenizer.tokenize("Godzilla Pancake red car day", Tokenization.Truncate.NONE, -1, 3).get(0) + tokenizer.tokenize("Elasticsearch", Tokenization.Truncate.NONE, -1, 0, null).get(0), + tokenizer.tokenize("my little red car", Tokenization.Truncate.NONE, -1, 1, null).get(0), + tokenizer.tokenize("Godzilla day", Tokenization.Truncate.NONE, -1, 2, null).get(0), + tokenizer.tokenize("Godzilla Pancake red car day", Tokenization.Truncate.NONE, -1, 3, null).get(0) ) ); assertThat(tr.getTokens(), hasSize(4)); @@ -754,7 +784,8 @@ public void testUnknownWordWithKnownSubWords() { new BertTokenization(null, false, null, Tokenization.Truncate.NONE, -1) ).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearchfoo fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearchfoo fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("[UNK]", "fun")); assertEquals(BertTokenizer.UNKNOWN_TOKEN, TEST_CASED_VOCAB.get(tokenization.tokenIds()[0])); assertEquals("fun", TEST_CASED_VOCAB.get(tokenization.tokenIds()[1])); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/MPNetTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/MPNetTokenizerTests.java index f15ff3a610a92..e21449f582a2d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/MPNetTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/MPNetTokenizerTests.java @@ -53,7 +53,8 @@ public void testTokenize() { new MPNetTokenization(null, false, null, Tokenization.Truncate.NONE, -1) ).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("Elastic", "##search", "fun")); assertArrayEquals(new int[] { 0, 1, 3 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 0, 1 }, tokenization.tokenMap()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizerTests.java index 202ca786d5207..5ec3afe55bedc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizerTests.java @@ -33,7 +33,8 @@ public void testTokenize() { new RobertaTokenization(true, false, null, Tokenization.Truncate.NONE, -1) ).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("Elast", "icsearch", "Ġfun")); assertArrayEquals(new int[] { 0, 297, 299, 275, 2 }, tokenization.tokenIds()); assertArrayEquals(new int[] { -1, 0, 0, 1, -1 }, tokenization.tokenMap()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizerTests.java index 7ba8602f7d290..c49a8fea15780 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizerTests.java @@ -73,7 +73,8 @@ public void testTokenize() throws IOException { new XLMRobertaTokenization(false, null, Tokenization.Truncate.NONE, -1) ).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁Ela", "stic", "search", "▁fun")); assertArrayEquals(new int[] { 4, 5, 6, 8 }, tokenization.tokenIds()); assertArrayEquals(new int[] { 0, 1, 2, 3 }, tokenization.tokenMap()); @@ -88,7 +89,8 @@ public void testTokenizeWithNeverSplit() throws IOException { new XLMRobertaTokenization(false, null, Tokenization.Truncate.NONE, -1) ).build() ) { - TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch ..", Tokenization.Truncate.NONE, -1, 0).get(0); + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch ..", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁Ela", "stic", "search", "▁", ".", "", "▁", ".")); } } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java index 92d46e54ea1cc..948b3a4b22ac1 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.LicensedFeature; @@ -66,6 +67,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -190,7 +192,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestMonitoringBulkAction(), new RestMonitoringMigrateAlertsAction()); } diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson index a830ef8da66f1..58e8281e1d32c 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson +++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson @@ -1,2 +1,4 @@ -{"create": {"_index": "profiling-hosts", "_id": "eLH27YsBj2lLi3tJYlvr"}} -{"profiling.project.id": 100, "host.id": "8457605156473051743", "@timestamp": 1700504426, "ecs.version": "1.12.0", "profiling.agent.build_timestamp": 1688111067, "profiling.instance.private_ipv4s": ["192.168.1.2"], "ec2.instance_life_cycle": "on-demand", "profiling.agent.config.map_scale_factor": 0, "ec2.instance_type": "i3.2xlarge", "profiling.host.ip": "192.168.1.2", "profiling.agent.config.bpf_log_level": 0, "profiling.host.sysctl.net.core.bpf_jit_enable": 1, "profiling.agent.config.file": "/etc/prodfiler/prodfiler.conf", "ec2.local_ipv4": "192.168.1.2", "profiling.agent.config.no_kernel_version_check": false, "profiling.host.machine": "x86_64", "profiling.host.tags": ["cloud_provider:aws", "cloud_environment:qa", "cloud_region:eu-west-1"], "profiling.agent.config.probabilistic_threshold": 100, "profiling.agent.config.disable_tls": false, "profiling.agent.config.tracers": "all", "profiling.agent.start_time": 1700090045589, "profiling.agent.config.max_elements_per_interval": 800, "ec2.placement.region": "eu-west-1", "profiling.agent.config.present_cpu_cores": 8, "profiling.host.kernel_version": "9.9.9-0-aws", "profiling.agent.config.bpf_log_size": 65536, "profiling.agent.config.known_traces_entries": 65536, "profiling.host.sysctl.kernel.unprivileged_bpf_disabled": 1, "profiling.agent.config.verbose": false, "profiling.agent.config.probabilistic_interval": "1m0s", "ec2.placement.availability_zone_id": "euw1-az1", "ec2.security_groups": "", "ec2.local_hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "ec2.placement.availability_zone": "eu-west-1c", "profiling.agent.config.upload_symbols": false, "profiling.host.sysctl.kernel.bpf_stats_enabled": 0, "profiling.host.name": "ip-192-168-1-2", "ec2.mac": "00:11:22:33:44:55", "profiling.host.kernel_proc_version": "Linux version 9.9.9-0-aws", "profiling.agent.config.cache_directory": "/var/cache/optimyze/", "profiling.agent.version": "v8.12.0", "ec2.hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "profiling.agent.config.elastic_mode": false, "ec2.ami_id": "ami-aaaaaaaaaaa", "ec2.instance_id": "i-0b999999999999999" } +{"create": {"_index": "profiling-hosts","_id":"eLH27YsBj2lLi3tJYlvr"}} +{"profiling.project.id":100,"host.id":"8457605156473051743","@timestamp":1700504426,"ecs.version":"1.12.0","profiling.agent.build_timestamp":1688111067,"profiling.instance.private_ipv4s":["192.168.1.2"],"ec2.instance_life_cycle":"on-demand","profiling.agent.config.map_scale_factor":0,"ec2.instance_type":"i3.2xlarge","profiling.host.ip":"192.168.1.2","profiling.agent.config.bpf_log_level":0,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.file":"/etc/prodfiler/prodfiler.conf","ec2.local_ipv4":"192.168.1.2","profiling.agent.config.no_kernel_version_check":false,"profiling.host.machine":"x86_64","profiling.host.tags":["cloud_provider:aws","cloud_environment:qa","cloud_region:eu-west-1"],"profiling.agent.config.probabilistic_threshold":100,"profiling.agent.config.disable_tls":false,"profiling.agent.config.tracers":"all","profiling.agent.start_time":1700090045589,"profiling.agent.config.max_elements_per_interval":800,"ec2.placement.region":"eu-west-1","profiling.agent.config.present_cpu_cores":8,"profiling.host.kernel_version":"9.9.9-0-aws","profiling.agent.config.bpf_log_size":65536,"profiling.agent.config.known_traces_entries":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.verbose":false,"profiling.agent.config.probabilistic_interval":"1m0s","ec2.placement.availability_zone_id":"euw1-az1","ec2.security_groups":"","ec2.local_hostname":"ip-192-168-1-2.eu-west-1.compute.internal","ec2.placement.availability_zone":"eu-west-1c","profiling.agent.config.upload_symbols":false,"profiling.host.sysctl.kernel.bpf_stats_enabled":0,"profiling.host.name":"ip-192-168-1-2","ec2.mac":"00:11:22:33:44:55","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-aws","profiling.agent.config.cache_directory":"/var/cache/optimyze/","profiling.agent.version":"v8.12.0","ec2.hostname":"ip-192-168-1-2.eu-west-1.compute.internal","profiling.agent.config.elastic_mode":false,"ec2.ami_id":"ami-aaaaaaaaaaa","ec2.instance_id":"i-0b999999999999999"} +{"create": {"_index": "profiling-hosts", "_id": "u_fHlYwBkmZvQ6tVo1Lr"}} +{"profiling.project.id":100,"host.id":"7416508186220657211","@timestamp":1703319912,"ecs.version":"1.12.0","profiling.agent.version":"8.11.0","profiling.agent.config.map_scale_factor":0,"profiling.agent.config.probabilistic_threshold":100,"profiling.host.name":"ip-192-186-1-3","profiling.agent.config.no_kernel_version_check":false,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.elastic_mode":false,"azure.compute.vmsize":"Standard_D4s_v3","azure.compute.environment":"AzurePublicCloud","profiling.agent.config.bpf_log_level":0,"profiling.agent.config.known_traces_entries":65536,"profiling.agent.config.ca_address":"example.com:443","profiling.agent.config.tags":"cloud_provider:azure;cloud_environment:qa;cloud_region:eastus2","profiling.host.tags":["cloud_provider:azure","cloud_environment:qa","cloud_region:eastus2"],"profiling.host.kernel_version":"9.9.9-0-azure","profiling.agent.revision":"head-52cc2030","azure.compute.subscriptionid":"1-2-3-4-5","profiling.host.sysctl.kernel.bpf_stats_enabled":0,"profiling.host.machine":"x86_64","azure.compute.zone":"3","profiling.agent.config.cache_directory":"/var/cache/Elastic/universal-profiling","azure.compute.name":"example-qa-eastus2-001-v1-zone3_6","profiling.agent.config.probabilistic_interval":"1m0s","azure.compute.location":"eastus2","azure.compute.version":"1234.20230510.233254","profiling.instance.private_ipv4s":["192.168.1.3"],"profiling.agent.build_timestamp":1699000836,"profiling.agent.config.file":"/etc/Elastic/universal-profiling/pf-host-agent.conf","profiling.agent.config.bpf_log_size":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.tracers":"all","profiling.agent.config.present_cpu_cores":4,"profiling.agent.start_time":1702306987358,"profiling.agent.config.disable_tls":false,"azure.compute.ostype":"Linux","profiling.host.ip":"192.168.1.3","profiling.agent.config.max_elements_per_interval":400,"profiling.agent.config.upload_symbols":false,"azure.compute.tags":"bootstrap-version:v1;ece-id:001;environment:qa;identifier:v1;initial-config:;managed-by:terraform;monitored-by:core-infrastructure;owner:core-infrastructure;region_type:ess;role:blueprint;secondary_role:;vars-identifier:eastus2-001-v1","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-azure","profiling.agent.config.verbose":false,"azure.compute.vmid":"1-2-3-4-5"} diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesRequest.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesRequest.java index 7672e2cc0c05b..ef7bcef36fd04 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesRequest.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesRequest.java @@ -36,7 +36,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque public static final ParseField QUERY_FIELD = new ParseField("query"); public static final ParseField SAMPLE_SIZE_FIELD = new ParseField("sample_size"); public static final ParseField INDICES_FIELD = new ParseField("indices"); - public static final ParseField STACKTRACE_IDS_FIELD = new ParseField("stacktrace_ids"); + public static final ParseField STACKTRACE_IDS_FIELD = new ParseField("stacktrace_ids_field"); public static final ParseField REQUESTED_DURATION_FIELD = new ParseField("requested_duration"); public static final ParseField AWS_COST_FACTOR_FIELD = new ParseField("aws_cost_factor"); public static final ParseField CUSTOM_CO2_PER_KWH = new ParseField("co2_per_kwh"); @@ -49,7 +49,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque private QueryBuilder query; private int sampleSize; private String indices; - private String stackTraceIds; + private String stackTraceIdsField; private Double requestedDuration; private Double awsCostFactor; private Double customCO2PerKWH; @@ -73,7 +73,7 @@ public GetStackTracesRequest( Double awsCostFactor, QueryBuilder query, String indices, - String stackTraceIds, + String stackTraceIdsField, Double customCO2PerKWH, Double customDatacenterPUE, Double customPerCoreWattX86, @@ -85,7 +85,7 @@ public GetStackTracesRequest( this.awsCostFactor = awsCostFactor; this.query = query; this.indices = indices; - this.stackTraceIds = stackTraceIds; + this.stackTraceIdsField = stackTraceIdsField; this.customCO2PerKWH = customCO2PerKWH; this.customDatacenterPUE = customDatacenterPUE; this.customPerCoreWattX86 = customPerCoreWattX86; @@ -138,8 +138,8 @@ public String getIndices() { return indices; } - public String getStackTraceIds() { - return stackTraceIds; + public String getStackTraceIdsField() { + return stackTraceIdsField; } public boolean isAdjustSampleCount() { @@ -170,7 +170,7 @@ public void parseXContent(XContentParser parser) throws IOException { } else if (INDICES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { this.indices = parser.text(); } else if (STACKTRACE_IDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - this.stackTraceIds = parser.text(); + this.stackTraceIdsField = parser.text(); } else if (REQUESTED_DURATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { this.requestedDuration = parser.doubleValue(); } else if (AWS_COST_FACTOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -215,14 +215,14 @@ public void parseXContent(XContentParser parser) throws IOException { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (indices != null) { - if (stackTraceIds == null || stackTraceIds.isEmpty()) { + if (stackTraceIdsField == null || stackTraceIdsField.isEmpty()) { validationException = addValidationError( "[" + STACKTRACE_IDS_FIELD.getPreferredName() + "] is mandatory", validationException ); } } else { - if (stackTraceIds != null) { + if (stackTraceIdsField != null) { validationException = addValidationError( "[" + STACKTRACE_IDS_FIELD.getPreferredName() + "] must not be set", validationException @@ -257,7 +257,7 @@ public String getDescription() { // generating description lazily since the query could be large StringBuilder sb = new StringBuilder(); appendField(sb, "indices", indices); - appendField(sb, "stacktrace_ids", stackTraceIds); + appendField(sb, "stacktrace_ids_field", stackTraceIdsField); appendField(sb, "sample_size", sampleSize); appendField(sb, "requested_duration", requestedDuration); appendField(sb, "aws_cost_factor", awsCostFactor); @@ -295,7 +295,7 @@ public boolean equals(Object o) { return Objects.equals(query, that.query) && Objects.equals(sampleSize, that.sampleSize) && Objects.equals(indices, that.indices) - && Objects.equals(stackTraceIds, that.stackTraceIds); + && Objects.equals(stackTraceIdsField, that.stackTraceIdsField); } @Override @@ -306,7 +306,7 @@ public int hashCode() { // Resampler to produce a consistent downsampling results, relying on the default hashCode implementation of `query` will // produce consistent results per node but not across the cluster. To avoid this, we produce the hashCode based on the // string representation instead, which will produce consistent results for the entire cluster and across node restarts. - return Objects.hash(Objects.toString(query, "null"), sampleSize, indices, stackTraceIds); + return Objects.hash(Objects.toString(query, "null"), sampleSize, indices, stackTraceIdsField); } @Override diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java index 150b2639e9ac3..ee649e381c85d 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java @@ -53,16 +53,18 @@ public static InstanceType fromHostSource(Map source) { region = tokens[0] + "-" + tokens[1]; } - // Support for instance type is planned for 8.13. + // Support for the instanceType name requires GCP data containing it. + // These are not publically available, so we don't support it for now. return new InstanceType("gcp", region, null); } // Check and handle Azure. + // example: "azure.compute.location": "eastus2" region = (String) source.get("azure.compute.location"); if (region != null) { - // example: "azure.compute.location": "eastus2" - // Support for instance type is planned for 8.13. - return new InstanceType("azure", region, null); + // example: "azure.compute.vmsize": "Standard_D2s_v3" + String instanceType = (String) source.get("azure.compute.vmsize"); + return new InstanceType("azure", region, instanceType); } // Support for configured tags (ECS). diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceTypeService.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceTypeService.java index 58dd19c91f966..3a1cad38f7781 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceTypeService.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceTypeService.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.profiling; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -20,44 +22,51 @@ import java.util.zip.GZIPInputStream; public final class InstanceTypeService { - private InstanceTypeService() {} - private static final class Holder { - private static final Map costsPerDatacenter; + private static final Logger log = LogManager.getLogger(InstanceTypeService.class); + private static final class Holder { + private static final Map COSTS_PER_DATACENTER; static { + final StopWatch watch = new StopWatch("loadProfilingCostsData"); + final Map tmp = new HashMap<>(); final Map objects = new HashMap<>(); final Function dedupString = s -> (String) objects.computeIfAbsent(s, Function.identity()); - final Map tmp = new HashMap<>(); - try ( - GZIPInputStream in = new GZIPInputStream( - InstanceTypeService.class.getClassLoader().getResourceAsStream("profiling-costs.json.gz") - ); - XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, in) - ) { - if (parser.currentToken() == null) { - parser.nextToken(); - } - List> rawData = XContentParserUtils.parseList(parser, XContentParser::map); - for (Map entry : rawData) { - tmp.put( - new InstanceType( - dedupString.apply((String) entry.get("provider")), - dedupString.apply((String) entry.get("region")), - dedupString.apply((String) entry.get("instance_type")) - ), - (CostEntry) objects.computeIfAbsent(CostEntry.fromSource(entry), Function.identity()) - ); + + // All files ar expected to exist. + for (String provider : List.of("aws", "azure")) { + String name = "profiling-costs-" + provider + ".json.gz"; + try ( + GZIPInputStream in = new GZIPInputStream(InstanceTypeService.class.getClassLoader().getResourceAsStream(name)); + XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, in) + ) { + if (parser.currentToken() == null) { + parser.nextToken(); + } + List> rawData = XContentParserUtils.parseList(parser, XContentParser::map); + for (Map entry : rawData) { + tmp.put( + new InstanceType( + provider, + dedupString.apply((String) entry.get("region")), + dedupString.apply((String) entry.get("instance_type")) + ), + (CostEntry) objects.computeIfAbsent(CostEntry.fromSource(entry), Function.identity()) + ); + } + } catch (IOException e) { + throw new ExceptionInInitializerError(e); } - costsPerDatacenter = Map.copyOf(tmp); - } catch (IOException e) { - throw new ExceptionInInitializerError(e); } + + COSTS_PER_DATACENTER = Map.copyOf(tmp); + + log.debug(watch::report); } } public static CostEntry getCosts(InstanceType instance) { - return Holder.costsPerDatacenter.get(instance); + return Holder.COSTS_PER_DATACENTER.get(instance); } } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java index c07d2a480b006..400ddfdbf73b6 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingPlugin.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -39,6 +40,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class ProfilingPlugin extends Plugin implements ActionPlugin { @@ -124,7 +126,8 @@ public List getRestHandlers( final IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter, final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster + final Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { List handlers = new ArrayList<>(); handlers.add(new RestGetStatusAction()); diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java index 33d568fbd0cdb..567c36e6b4404 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation; +import org.elasticsearch.search.aggregations.bucket.countedterms.CountedTermsAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.sampler.random.RandomSamplerAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; @@ -48,7 +49,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; -import org.elasticsearch.xpack.countedkeyword.CountedTermsAggregationBuilder; import java.time.Duration; import java.time.Instant; @@ -266,7 +266,8 @@ private void searchGenericEventGroupedByStackTrace( new RandomSamplerAggregationBuilder("sample").setSeed(request.hashCode()) .setProbability(responseBuilder.getSamplingRate()) .subAggregation( - new CountedTermsAggregationBuilder("group_by").size(MAX_TRACE_EVENTS_RESULT_SIZE).field(request.getStackTraceIds()) + new CountedTermsAggregationBuilder("group_by").size(MAX_TRACE_EVENTS_RESULT_SIZE) + .field(request.getStackTraceIdsField()) ) ) .execute(handleEventsGroupedByStackTrace(submitTask, client, responseBuilder, submitListener, searchResponse -> { diff --git a/x-pack/plugin/profiling/src/main/resources/profiling-costs-aws.json.gz b/x-pack/plugin/profiling/src/main/resources/profiling-costs-aws.json.gz new file mode 100644 index 0000000000000..b83ace66a5eb2 Binary files /dev/null and b/x-pack/plugin/profiling/src/main/resources/profiling-costs-aws.json.gz differ diff --git a/x-pack/plugin/profiling/src/main/resources/profiling-costs-azure.json.gz b/x-pack/plugin/profiling/src/main/resources/profiling-costs-azure.json.gz new file mode 100644 index 0000000000000..a760825bfe7e9 Binary files /dev/null and b/x-pack/plugin/profiling/src/main/resources/profiling-costs-azure.json.gz differ diff --git a/x-pack/plugin/profiling/src/main/resources/profiling-costs.json.gz b/x-pack/plugin/profiling/src/main/resources/profiling-costs.json.gz deleted file mode 100644 index 590c0ff606201..0000000000000 Binary files a/x-pack/plugin/profiling/src/main/resources/profiling-costs.json.gz and /dev/null differ diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/CostCalculatorTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/CostCalculatorTests.java index 185451d0a9235..f100a3e57b50c 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/CostCalculatorTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/CostCalculatorTests.java @@ -12,26 +12,42 @@ import java.util.Map; public class CostCalculatorTests extends ESTestCase { - private static final String HOST_ID_A = "1110256254710195391"; - private static final String HOST_ID_B = "2220256254710195392"; + private static final String HOST_ID_AWS = "1110256254710195391"; + private static final int HOST_ID_A_NUM_CORES = 8; + private static final String HOST_ID_AZURE = "2220256254710195392"; + private static final int HOST_ID_AZURE_NUM_CORES = 8; + private static final String HOST_ID_UNKNOWN = "3330256254710195392"; + private static final Integer HOST_ID_UNKNOWN_NUM_CORES = null; // number of cores are unknown public void testCreateFromRegularSource() { // tag::noformat Map hostsTable = Map.ofEntries( - Map.entry(HOST_ID_A, + Map.entry(HOST_ID_AWS, // known datacenter - new HostMetadata(HOST_ID_A, + new HostMetadata(HOST_ID_AWS, new InstanceType( "aws", "eu-west-1", "c5n.xlarge" ), "", // Doesn't matter for cost calculation. - null + HOST_ID_A_NUM_CORES // number of cores ) ), - Map.entry(HOST_ID_B, - new HostMetadata(HOST_ID_B, + Map.entry(HOST_ID_AZURE, + // known datacenter + new HostMetadata(HOST_ID_AZURE, + new InstanceType( + "azure", + "eastus2", + "Standard_D4s_v3" + ), + "", // Doesn't matter for cost calculation. + HOST_ID_AZURE_NUM_CORES // number of cores + ) + ), + Map.entry(HOST_ID_UNKNOWN, + new HostMetadata(HOST_ID_UNKNOWN, // unknown datacenter new InstanceType( "on-prem-provider", @@ -39,7 +55,7 @@ public void testCreateFromRegularSource() { "on-prem-instance-type" ), "", // Doesn't matter for cost calculation. - null + HOST_ID_UNKNOWN_NUM_CORES // number of cores ) ) ); @@ -50,15 +66,25 @@ public void testCreateFromRegularSource() { double annualCoreHours = CostCalculator.annualCoreHours(samplingDurationInSeconds, samples, 20.0d); CostCalculator costCalculator = new CostCalculator(hostsTable, samplingDurationInSeconds, null, null); - // Checks whether the cost calculation is based on the pre-calculated lookup data. - checkCostCalculation(costCalculator.annualCostsUSD(HOST_ID_A, samples), annualCoreHours, 0.061d); + // Checks whether the cost calculation is based on the lookup data. + // The usd_per_hour value can be looked up from profiling-costs-aws.json.gz. + checkCostCalculation(costCalculator.annualCostsUSD(HOST_ID_AWS, samples), annualCoreHours, 0.244d, HOST_ID_A_NUM_CORES); - // Checks whether the cost calculation is based on the default cost factor. - checkCostCalculation(costCalculator.annualCostsUSD(HOST_ID_B, samples), annualCoreHours, 0.0425d); + // Checks whether the cost calculation is based on the lookup data. + // The usd_per_hour value can be looked up from profiling-costs-azure.json.gz. + checkCostCalculation(costCalculator.annualCostsUSD(HOST_ID_AZURE, samples), annualCoreHours, 0.192d, HOST_ID_A_NUM_CORES); + + // Checks whether the cost calculation is based on the default values. + checkCostCalculation( + costCalculator.annualCostsUSD(HOST_ID_UNKNOWN, samples), + annualCoreHours, + CostCalculator.DEFAULT_COST_USD_PER_CORE_HOUR * HostMetadata.DEFAULT_PROFILING_NUM_CORES, + HostMetadata.DEFAULT_PROFILING_NUM_CORES + ); } - private void checkCostCalculation(double calculatedAnnualCostsUSD, double annualCoreHours, double costFactor) { - double expectedAnnualCostsUSD = annualCoreHours * costFactor; + private void checkCostCalculation(double calculatedAnnualCostsUSD, double annualCoreHours, double usd_per_hour, int profilingNumCores) { + double expectedAnnualCostsUSD = annualCoreHours * (usd_per_hour / profilingNumCores); assertEquals(expectedAnnualCostsUSD, calculatedAnnualCostsUSD, 0.00000001d); } } diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/GetStackTracesRequestTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/GetStackTracesRequestTests.java index 9594bd5233a5e..d4dbb1eedef51 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/GetStackTracesRequestTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/GetStackTracesRequestTests.java @@ -50,7 +50,7 @@ public void testParseValidXContent() throws IOException { assertEquals("@timestamp", ((RangeQueryBuilder) request.getQuery()).fieldName()); // Expect the default values assertNull(request.getIndices()); - assertNull(request.getStackTraceIds()); + assertNull(request.getStackTraceIdsField()); assertNull(request.getAwsCostFactor()); assertNull(request.getCustomCO2PerKWH()); assertNull(request.getCustomDatacenterPUE()); @@ -66,7 +66,7 @@ public void testParseValidXContentWithCustomIndex() throws IOException { .startObject() .field("sample_size", 2000) .field("indices", "my-traces") - .field("stacktrace_ids", "stacktraces") + .field("stacktrace_ids_field", "stacktraces") .startObject("query") .startObject("range") .startObject("@timestamp") @@ -83,7 +83,7 @@ public void testParseValidXContentWithCustomIndex() throws IOException { assertEquals(2000, request.getSampleSize()); assertEquals("my-traces", request.getIndices()); - assertEquals("stacktraces", request.getStackTraceIds()); + assertEquals("stacktraces", request.getStackTraceIdsField()); // a basic check suffices here assertEquals("@timestamp", ((RangeQueryBuilder) request.getQuery()).fieldName()); @@ -138,7 +138,7 @@ public void testParseValidXContentWithCustomCostAndCO2Data() throws IOException // Expect the default values assertNull(request.getIndices()); - assertNull(request.getStackTraceIds()); + assertNull(request.getStackTraceIdsField()); } } @@ -217,7 +217,7 @@ public void testValidateStacktraceWithoutIndices() { ); List validationErrors = request.validate().validationErrors(); assertEquals(1, validationErrors.size()); - assertEquals("[stacktrace_ids] must not be set", validationErrors.get(0)); + assertEquals("[stacktrace_ids_field] must not be set", validationErrors.get(0)); } public void testValidateIndicesWithoutStacktraces() { @@ -236,7 +236,7 @@ public void testValidateIndicesWithoutStacktraces() { ); List validationErrors = request.validate().validationErrors(); assertEquals(1, validationErrors.size()); - assertEquals("[stacktrace_ids] is mandatory", validationErrors.get(0)); + assertEquals("[stacktrace_ids_field] is mandatory", validationErrors.get(0)); } public void testConsidersCustomIndicesInRelatedIndices() { diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/RepositoriesMeteringPlugin.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/RepositoriesMeteringPlugin.java index 4441ec70f74aa..7532fe974830c 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/RepositoriesMeteringPlugin.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/RepositoriesMeteringPlugin.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -28,6 +29,7 @@ import org.elasticsearch.xpack.repositories.metering.rest.RestGetRepositoriesMeteringAction; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public final class RepositoriesMeteringPlugin extends Plugin implements ActionPlugin { @@ -49,7 +51,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestGetRepositoriesMeteringAction(), new RestClearRepositoriesMeteringArchiveAction()); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java index 39e68a11c0d59..41cca2a219ff4 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; @@ -62,6 +63,7 @@ import java.time.Clock; import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class Rollup extends Plugin implements ActionPlugin, PersistentTaskPlugin { @@ -93,7 +95,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList( new RestRollupSearchAction(namedWriteableRegistry), diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index b08f31083c973..6ce448bdd63d5 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -42,6 +42,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; @@ -507,7 +508,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of( new RestSearchableSnapshotsStatsAction(), diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java index 52d87c2e32c87..aef0ec95372cf 100644 --- a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java +++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java @@ -542,15 +542,12 @@ public void testReloadClientSecret() throws Exception { // secret updated, so authentication succeeds getSecurityClient(buildAndSignJwtForRealm2(principal), Optional.of(newValidSharedSecret)).authenticate(); - // removing setting also works and leads to authentication failure + // removing setting should not work since it can + // lead to inconsistency in realm's configuration + // and eventual authentication failures writeSettingToKeystoreThenReload("xpack.security.authc.realms.jwt.jwt2.client_authentication.shared_secret", null); - assertThat( - expectThrows( - ResponseException.class, - () -> getSecurityClient(buildAndSignJwtForRealm2(principal), Optional.of(newValidSharedSecret)).authenticate() - ).getResponse(), - hasStatusCode(RestStatus.UNAUTHORIZED) - ); + getSecurityClient(buildAndSignJwtForRealm2(principal), Optional.of(newValidSharedSecret)).authenticate(); + } finally { // Restore setting for other tests writeSettingToKeystoreThenReload( diff --git a/x-pack/plugin/security/qa/multi-cluster/build.gradle b/x-pack/plugin/security/qa/multi-cluster/build.gradle index 993c0bf6c6e18..625b6806ab520 100644 --- a/x-pack/plugin/security/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/security/qa/multi-cluster/build.gradle @@ -20,6 +20,10 @@ dependencies { clusterModules project(':x-pack:plugin:ccr') clusterModules(project(":modules:reindex")) // need for deleting transform jobs clusterModules(project(":x-pack:plugin:transform")) + // esql with enrich + clusterModules project(':x-pack:plugin:esql') + clusterModules project(':x-pack:plugin:enrich') + clusterModules(project(":modules:ingest-common")) } tasks.named("javaRestTest") { diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java new file mode 100644 index 0000000000000..e181a3542d446 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java @@ -0,0 +1,437 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.Build; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Strings; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.junit.RunnableTestRuleAdapter; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTestCase { + + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + private static final AtomicReference> REST_API_KEY_MAP_REF = new AtomicReference<>(); + private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); + private static final AtomicBoolean NODE1_RCS_SERVER_ENABLED = new AtomicBoolean(); + private static final AtomicBoolean NODE2_RCS_SERVER_ENABLED = new AtomicBoolean(); + private static final AtomicInteger INVALID_SECRET_LENGTH = new AtomicInteger(); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .nodes(3) + .module("x-pack-esql") + .module("x-pack-enrich") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .node(1, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE1_RCS_SERVER_ENABLED.get()))) + .node(2, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE2_RCS_SERVER_ENABLED.get()))) + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["index*", "not_found_index", "employees"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + // Define a bogus API key for another remote cluster + .keystore("cluster.remote.invalid_remote.credentials", randomEncodedApiKey()) + // Define remote with a REST API key to observe expected failure + .keystore("cluster.remote.wrong_api_key_type.credentials", () -> { + if (REST_API_KEY_MAP_REF.get() == null) { + initFulfillingClusterClient(); + final var createApiKeyRequest = new Request("POST", "/_security/api_key"); + createApiKeyRequest.setJsonEntity(""" + { + "name": "rest_api_key" + }"""); + try { + final Response createApiKeyResponse = performRequestWithAdminUser(fulfillingClusterClient, createApiKeyRequest); + assertOK(createApiKeyResponse); + REST_API_KEY_MAP_REF.set(responseAsMap(createApiKeyResponse)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return (String) REST_API_KEY_MAP_REF.get().get("encoded"); + }) + // Define a remote with invalid API key secret length + .keystore( + "cluster.remote.invalid_secret_length.credentials", + () -> Base64.getEncoder() + .encodeToString( + (UUIDs.base64UUID() + ":" + randomAlphaOfLength(INVALID_SECRET_LENGTH.get())).getBytes(StandardCharsets.UTF_8) + ) + ) + .rolesFile(Resource.fromClasspath("roles.yml")) + .user(REMOTE_METRIC_USER, PASS.toString(), "read_remote_shared_metrics", false) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + // `SSL_ENABLED_REF` is used to control the SSL-enabled setting on the test clusters + // We set it here, since randomization methods are not available in the static initialize context above + public static TestRule clusterRule = RuleChain.outerRule(new RunnableTestRuleAdapter(() -> { + SSL_ENABLED_REF.set(usually()); + NODE1_RCS_SERVER_ENABLED.set(randomBoolean()); + NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); + INVALID_SECRET_LENGTH.set(randomValueOtherThan(22, () -> randomIntBetween(0, 99))); + })).around(fulfillingCluster).around(queryCluster); + + public void populateData() throws Exception { + CheckedConsumer setupEnrich = client -> { + Request createIndex = new Request("PUT", "countries"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "emp_id": { "type": "keyword" }, + "country": { "type": "text" } + } + } + } + """); + assertOK(performRequestWithAdminUser(client, createIndex)); + final Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(Strings.format(""" + { "index": { "_index": "countries" } } + { "emp_id": "1", "country": "usa"} + { "index": { "_index": "countries" } } + { "emp_id": "2", "country": "canada"} + { "index": { "_index": "countries" } } + { "emp_id": "3", "country": "germany"} + { "index": { "_index": "countries" } } + { "emp_id": "4", "country": "spain"} + { "index": { "_index": "countries" } } + { "emp_id": "5", "country": "japan"} + { "index": { "_index": "countries" } } + { "emp_id": "6", "country": "france"} + { "index": { "_index": "countries" } } + { "emp_id": "7", "country": "usa"} + { "index": { "_index": "countries" } } + { "emp_id": "8", "country": "canada"} + { "index": { "_index": "countries" } } + { "emp_id": "9", "country": "usa"} + """)); + assertOK(performRequestWithAdminUser(client, bulkRequest)); + + Request createEnrich = new Request("PUT", "/_enrich/policy/countries"); + createEnrich.setJsonEntity(""" + { + "match": { + "indices": "countries", + "match_field": "emp_id", + "enrich_fields": ["country"] + } + } + """); + assertOK(performRequestWithAdminUser(client, createEnrich)); + assertOK(performRequestWithAdminUser(client, new Request("PUT", "_enrich/policy/countries/_execute"))); + performRequestWithAdminUser(client, new Request("DELETE", "/countries")); + }; + // Fulfilling cluster + { + setupEnrich.accept(fulfillingClusterClient); + Request createIndex = new Request("PUT", "employees"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "emp_id": { "type": "keyword" }, + "department": {"type": "keyword" } + } + } + } + """); + assertOK(performRequestAgainstFulfillingCluster(createIndex)); + final Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(Strings.format(""" + { "index": { "_index": "employees" } } + { "emp_id": "1", "department" : "engineering" } + { "index": { "_index": "employees" } } + { "emp_id": "3", "department" : "sales" } + { "index": { "_index": "employees" } } + { "emp_id": "5", "department" : "marketing" } + { "index": { "_index": "employees" } } + { "emp_id": "7", "department" : "engineering" } + { "index": { "_index": "employees" } } + { "emp_id": "9", "department" : "sales" } + """)); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + } + // Querying cluster + // Index some documents, to use them in a mixed-cluster search + setupEnrich.accept(client()); + Request createIndex = new Request("PUT", "employees"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "emp_id": { "type": "keyword" }, + "department": { "type": "keyword" } + } + } + } + """); + assertOK(adminClient().performRequest(createIndex)); + final Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(Strings.format(""" + { "index": { "_index": "employees" } } + { "emp_id": "2", "department" : "management" } + { "index": { "_index": "employees"} } + { "emp_id": "4", "department" : "engineering" } + { "index": { "_index": "employees" } } + { "emp_id": "6", "department" : "marketing"} + { "index": { "_index": "employees"} } + { "emp_id": "8", "department" : "support"} + """)); + assertOK(client().performRequest(bulkRequest)); + + // Create user role with privileges for remote and local indices + final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); + putRoleRequest.setJsonEntity(""" + { + "indices": [ + { + "names": ["employees"], + "privileges": ["read"] + } + ], + "cluster": [ "monitor_enrich" ], + "remote_indices": [ + { + "names": ["employees"], + "privileges": ["read", "read_cross_cluster"], + "clusters": ["my_remote_cluster"] + } + ] + }"""); + assertOK(adminClient().performRequest(putRoleRequest)); + final var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER); + putUserRequest.setJsonEntity(""" + { + "password": "x-pack-test-password", + "roles" : ["remote_search"] + }"""); + assertOK(adminClient().performRequest(putUserRequest)); + } + + @After + public void wipeData() throws Exception { + CheckedConsumer wipe = client -> { + performRequestWithAdminUser(client, new Request("DELETE", "/employees")); + performRequestWithAdminUser(client, new Request("DELETE", "/_enrich/policy/countries")); + }; + wipe.accept(fulfillingClusterClient); + wipe.accept(client()); + } + + @AwaitsFix(bugUrl = "cross-clusters query doesn't work with RCS 2.0") + public void testCrossClusterQuery() throws Exception { + configureRemoteCluster(); + populateData(); + // Query cluster + { + { + Response response = performRequestWithRemoteSearchUser(esqlRequest(""" + FROM my_remote_cluster:employees + | SORT emp_id ASC + | LIMIT 2 + | KEEP emp_id, department""")); + assertOK(response); + Map values = entityAsMap(response); + } + { + Response response = performRequestWithRemoteSearchUser(esqlRequest(""" + FROM my_remote_cluster:employees,employees + | SORT emp_id ASC + | LIMIT 10""")); + assertOK(response); + + } + // Check that authentication fails if we use a non-existent API key + updateClusterSettings( + randomBoolean() + ? Settings.builder() + .put("cluster.remote.invalid_remote.seeds", fulfillingCluster.getRemoteClusterServerEndpoint(0)) + .build() + : Settings.builder() + .put("cluster.remote.invalid_remote.mode", "proxy") + .put("cluster.remote.invalid_remote.proxy_address", fulfillingCluster.getRemoteClusterServerEndpoint(0)) + .build() + ); + for (String indices : List.of("my_remote_cluster:employees,employees", "my_remote_cluster:employees")) { + ResponseException error = expectThrows(ResponseException.class, () -> { + var q = "FROM " + indices + "| SORT emp_id DESC | LIMIT 10"; + performRequestWithLocalSearchUser(esqlRequest(q)); + }); + assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(403)); + assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(401)); + assertThat(error.getMessage(), containsString("unable to find apikey")); + } + } + } + + @AwaitsFix(bugUrl = "cross-clusters enrich doesn't work with RCS 2.0") + public void testCrossClusterEnrich() throws Exception { + configureRemoteCluster(); + populateData(); + // Query cluster + { + // ESQL with enrich is okay when user has access to enrich polices + Response response = performRequestWithRemoteSearchUser(esqlRequest(""" + FROM my_remote_cluster:employees,employees + | ENRICH countries + | STATS size=count(*) by country + | SORT size DESC + | LIMIT 2""")); + assertOK(response); + Map values = entityAsMap(response); + + // ESQL with enrich is denied when user has no access to enrich policies + final var putLocalSearchRoleRequest = new Request("PUT", "/_security/role/local_search"); + putLocalSearchRoleRequest.setJsonEntity(""" + { + "indices": [ + { + "names": ["employees"], + "privileges": ["read"] + } + ], + "cluster": [ ], + "remote_indices": [ + { + "names": ["employees"], + "privileges": ["read", "read_cross_cluster"], + "clusters": ["my_remote_cluster"] + } + ] + }"""); + assertOK(adminClient().performRequest(putLocalSearchRoleRequest)); + final var putlocalSearchUserRequest = new Request("PUT", "/_security/user/local_search_user"); + putlocalSearchUserRequest.setJsonEntity(""" + { + "password": "x-pack-test-password", + "roles" : ["local_search"] + }"""); + assertOK(adminClient().performRequest(putlocalSearchUserRequest)); + for (String indices : List.of("my_remote_cluster:employees,employees", "my_remote_cluster:employees")) { + ResponseException error = expectThrows(ResponseException.class, () -> { + var q = "FROM " + indices + "| ENRICH countries | STATS size=count(*) by country | SORT size | LIMIT 2"; + performRequestWithLocalSearchUser(esqlRequest(q)); + }); + assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(403)); + assertThat( + error.getMessage(), + containsString( + "action [cluster:monitor/xpack/enrich/esql/resolve_policy] towards remote cluster [my_remote_cluster]" + + " is unauthorized for user [local_search_user] with effective roles [local_search]" + ) + ); + } + } + } + + protected Request esqlRequest(String command) throws IOException { + XContentBuilder body = JsonXContent.contentBuilder(); + body.startObject(); + body.field("query", command); + if (Build.current().isSnapshot() && randomBoolean()) { + Settings.Builder settings = Settings.builder(); + if (randomBoolean()) { + settings.put("page_size", between(1, 5)); + } + if (randomBoolean()) { + settings.put("exchange_buffer_size", between(1, 2)); + } + if (randomBoolean()) { + settings.put("data_partitioning", randomFrom("shard", "segment", "doc")); + } + if (randomBoolean()) { + settings.put("enrich_max_workers", between(1, 5)); + } + Settings pragmas = settings.build(); + if (pragmas != Settings.EMPTY) { + body.startObject("pragma"); + body.value(pragmas); + body.endObject(); + } + } + body.endObject(); + Request request = new Request("POST", "_query"); + request.setJsonEntity(org.elasticsearch.common.Strings.toString(body)); + return request; + } + + private Response performRequestWithRemoteSearchUser(final Request request) throws IOException { + request.setOptions( + RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(REMOTE_SEARCH_USER, PASS)) + ); + return client().performRequest(request); + } + + private Response performRequestWithLocalSearchUser(final Request request) throws IOException { + request.setOptions( + RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod("local_search_user", PASS)) + ); + return client().performRequest(request); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index b6893e853f256..15477f8a1536b 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -128,9 +128,11 @@ public class Constants { "cluster:admin/xpack/connector/list", "cluster:admin/xpack/connector/post", "cluster:admin/xpack/connector/put", + "cluster:admin/xpack/connector/update_api_key_id", "cluster:admin/xpack/connector/update_configuration", "cluster:admin/xpack/connector/update_error", "cluster:admin/xpack/connector/update_filtering", + "cluster:admin/xpack/connector/update_index_name", "cluster:admin/xpack/connector/update_last_seen", "cluster:admin/xpack/connector/update_last_sync_stats", "cluster:admin/xpack/connector/update_name", @@ -138,6 +140,7 @@ public class Constants { "cluster:admin/xpack/connector/update_pipeline", "cluster:admin/xpack/connector/update_scheduling", "cluster:admin/xpack/connector/update_service_type", + "cluster:admin/xpack/connector/update_status", "cluster:admin/xpack/connector/secret/delete", "cluster:admin/xpack/connector/secret/get", "cluster:admin/xpack/connector/secret/post", diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesTestPlugin.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesTestPlugin.java index 87ef55b5b8633..16dd7b4bb2b3c 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesTestPlugin.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesTestPlugin.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -21,6 +22,7 @@ import org.elasticsearch.xpack.security.operator.actions.RestGetActionsAction; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class OperatorPrivilegesTestPlugin extends Plugin implements ActionPlugin { @@ -34,7 +36,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestGetActionsAction()); } diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 49c2da7b173ec..4e639e14eda6e 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -460,6 +460,33 @@ public void testBulkUpdateExpirationTimeApiKey() throws IOException { assertEquals(List.of(), response.get("noops")); } + public void testUpdateBadExpirationTimeApiKey() throws IOException { + final EncodedApiKey apiKey = createApiKey("my-api-key-name", Map.of()); + + final boolean bulkUpdate = randomBoolean(); + TimeValue expiration = randomFrom(TimeValue.ZERO, TimeValue.MINUS_ONE); + final String method; + final Map requestBody; + final String requestPath; + + if (bulkUpdate) { + method = "POST"; + requestBody = Map.of("expiration", expiration, "ids", List.of(apiKey.id)); + requestPath = "_security/api_key/_bulk_update"; + } else { + method = "PUT"; + requestBody = Map.of("expiration", expiration); + requestPath = "_security/api_key/" + apiKey.id; + } + + final var bulkUpdateApiKeyRequest = new Request(method, requestPath); + bulkUpdateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString()); + + final ResponseException e = expectThrows(ResponseException.class, () -> adminClient().performRequest(bulkUpdateApiKeyRequest)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + assertThat(e.getMessage(), containsString("API key expiration must be in the future")); + } + public void testGrantTargetCanUpdateApiKey() throws IOException { final var request = new Request("POST", "_security/api_key/grant"); request.setOptions( diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index 875026c02754f..745ea34e8eb89 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.ReleasableRef; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -1148,24 +1149,28 @@ private static MockTransportService startTransport( capturedHeaders.add( new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) ); - channel.sendResponse( - new SearchResponse( - SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + try ( + var searchResponseRef = ReleasableRef.of( + new SearchResponse( + SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + false, + null, + null, + 1, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) ) - ); + ) { + channel.sendResponse(searchResponseRef.get()); + } } ); service.start(); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index faa85150dca31..3025e3f061fcd 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -593,6 +593,7 @@ public void testRefreshingMultipleTimesFails() throws Exception { assertThat(e, throwableWithMessage(containsString("token has already been refreshed more than 30 seconds in the past"))); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/85697") public void testRefreshingMultipleTimesWithinWindowSucceeds() throws Exception { final Clock clock = Clock.systemUTC(); final List tokens = Collections.synchronizedList(new ArrayList<>()); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java index c9b43afd4322d..ac033ba75798a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java @@ -16,6 +16,7 @@ import com.nimbusds.jwt.SignedJWT; import org.apache.http.HttpEntity; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -61,10 +62,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; +import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.CLIENT_AUTHENTICATION_TYPE; import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.CLIENT_AUTH_SHARED_SECRET_ROTATION_GRACE_PERIOD; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -124,7 +127,20 @@ protected Settings nodeSettings() { .put("xpack.security.authc.realms.jwt.jwt2.claims.groups", "groups") .put("xpack.security.authc.realms.jwt.jwt2.client_authentication.type", "shared_secret") .put("xpack.security.authc.realms.jwt.jwt2.client_authentication.rotation_grace_period", "0s") - .putList("xpack.security.authc.realms.jwt.jwt2.allowed_signature_algorithms", "HS256", "HS384"); + .putList("xpack.security.authc.realms.jwt.jwt2.allowed_signature_algorithms", "HS256", "HS384") + // 4th JWT realm + .put("xpack.security.authc.realms.jwt.jwt3.order", 40) + .put("xpack.security.authc.realms.jwt.jwt3.token_type", "id_token") + .put("xpack.security.authc.realms.jwt.jwt3.allowed_issuer", "my-issuer-04") + .put("xpack.security.authc.realms.jwt.jwt3.allowed_subjects", "user-04") + .put("xpack.security.authc.realms.jwt.jwt3.allowed_audiences", "es-04") + .put("xpack.security.authc.realms.jwt.jwt3.claims.principal", "sub") + .put("xpack.security.authc.realms.jwt.jwt3.claims.groups", "groups") + .put("xpack.security.authc.realms.jwt.jwt3.client_authentication.type", "NONE") + .put( + "xpack.security.authc.realms.jwt.jwt3.pkc_jwkset_path", + getDataPath("/org/elasticsearch/xpack/security/authc/apikey/rsa-public-jwkset.json") + ); SecuritySettingsSource.addSecureSettings(builder, secureSettings -> { secureSettings.setString("xpack.security.authc.realms.jwt.jwt0.hmac_key", jwtHmacKey); @@ -491,6 +507,103 @@ public void testClientSecretRotation() throws Exception { } } + public void testValidationDuringReloadingClientSecrets() { + final Map realmsByName = getJwtRealms().stream().collect(Collectors.toMap(Realm::name, r -> r)); + final Set realmsWithSharedSecret = Set.of(realmsByName.get("jwt0"), realmsByName.get("jwt1"), realmsByName.get("jwt2")); + final JwtRealm realmWithoutSharedSecret = realmsByName.get("jwt3"); + + // Sanity check all client_authentication.type settings. + for (JwtRealm realm : realmsWithSharedSecret) { + assertThat(getClientAuthenticationType(realm), equalTo(JwtRealmSettings.ClientAuthenticationType.SHARED_SECRET)); + } + assertThat(getClientAuthenticationType(realmWithoutSharedSecret), equalTo(JwtRealmSettings.ClientAuthenticationType.NONE)); + + // Randomly chose one JWT realm which requires shared secret and omit it. + final MockSecureSettings newSecureSettings = new MockSecureSettings(); + final JwtRealm chosenRealmToRemoveSharedSecret = randomFrom(realmsWithSharedSecret); + for (JwtRealm realm : realmsWithSharedSecret) { + if (realm != chosenRealmToRemoveSharedSecret) { + newSecureSettings.setString( + "xpack.security.authc.realms.jwt." + realm.name() + ".client_authentication.shared_secret", + realm.name() + "_shared_secret" + ); + } + } + + // Reload settings and check if validation prevented updating for randomly chosen realm. + final PluginsService plugins = getInstanceFromNode(PluginsService.class); + final LocalStateSecurity localStateSecurity = plugins.filterPlugins(LocalStateSecurity.class).findFirst().get(); + final Security securityPlugin = localStateSecurity.plugins() + .stream() + .filter(p -> p instanceof Security) + .map(Security.class::cast) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Security plugin not found!")); + + Settings.Builder newSettingsBuilder = Settings.builder().setSecureSettings(newSecureSettings); + { + var e = expectThrows(ElasticsearchException.class, () -> securityPlugin.reload(newSettingsBuilder.build())); + assertThat(e.getMessage(), containsString("secure settings reload failed for one or more security component")); + + var suppressedExceptions = e.getSuppressed(); + assertThat(suppressedExceptions.length, equalTo(1)); + assertThat(suppressedExceptions[0].getMessage(), containsString("secure settings reload failed for one or more realms")); + + var realmSuppressedExceptions = suppressedExceptions[0].getSuppressed(); + assertThat(realmSuppressedExceptions.length, equalTo(1)); + assertThat( + realmSuppressedExceptions[0].getMessage(), + containsString( + "Missing setting for [xpack.security.authc.realms.jwt." + + chosenRealmToRemoveSharedSecret.name() + + ".client_authentication.shared_secret]. It is required when setting [xpack.security.authc.realms.jwt." + + chosenRealmToRemoveSharedSecret.name() + + ".client_authentication.type] is [" + + JwtRealmSettings.ClientAuthenticationType.SHARED_SECRET.value() + + "]" + ) + ); + } + + // Add missing required shared secret setting in order + // to avoid raising an exception for realm which has + // client_authentication.type set to shared_secret. + newSecureSettings.setString( + "xpack.security.authc.realms.jwt." + chosenRealmToRemoveSharedSecret.name() + ".client_authentication.shared_secret", + chosenRealmToRemoveSharedSecret.name() + "_shared_secret" + ); + // Add shared secret for realm which does not require it, + // because it has client_authentication.type set to NONE. + newSecureSettings.setString( + "xpack.security.authc.realms.jwt." + realmWithoutSharedSecret.name() + ".client_authentication.shared_secret", + realmWithoutSharedSecret.name() + "_shared_secret" + ); + + { + var e = expectThrows(ElasticsearchException.class, () -> securityPlugin.reload(newSettingsBuilder.build())); + assertThat(e.getMessage(), containsString("secure settings reload failed for one or more security component")); + + var suppressedExceptions = e.getSuppressed(); + assertThat(suppressedExceptions.length, equalTo(1)); + assertThat(suppressedExceptions[0].getMessage(), containsString("secure settings reload failed for one or more realms")); + + var realmSuppressedExceptions = suppressedExceptions[0].getSuppressed(); + assertThat(realmSuppressedExceptions.length, equalTo(1)); + assertThat( + realmSuppressedExceptions[0].getMessage(), + containsString( + "Setting [xpack.security.authc.realms.jwt." + + realmWithoutSharedSecret.name() + + ".client_authentication.shared_secret] is not supported, because setting [xpack.security.authc.realms.jwt." + + realmWithoutSharedSecret.name() + + ".client_authentication.type] is [" + + JwtRealmSettings.ClientAuthenticationType.NONE.value() + + "]" + ) + ); + } + } + private SignedJWT getSignedJWT(JWTClaimsSet claimsSet, byte[] hmacKeyBytes) throws Exception { JWSHeader jwtHeader = new JWSHeader.Builder(JWSAlgorithm.HS256).build(); OctetSequenceKey.Builder jwt0signer = new OctetSequenceKey.Builder(hmacKeyBytes); @@ -517,6 +630,10 @@ private TimeValue getGracePeriod(JwtRealm realm) { return realm.getConfig().getConcreteSetting(CLIENT_AUTH_SHARED_SECRET_ROTATION_GRACE_PERIOD).get(realm.getConfig().settings()); } + private JwtRealmSettings.ClientAuthenticationType getClientAuthenticationType(JwtRealm realm) { + return realm.getConfig().getConcreteSetting(CLIENT_AUTHENTICATION_TYPE).get(realm.getConfig().settings()); + } + private void assertJwtToken(JwtAuthenticationToken token, String tokenPrincipal, String sharedSecret, SignedJWT signedJWT) throws ParseException { assertThat(token.principal(), equalTo(tokenPrincipal)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 57fc1b319fa8a..c59ccd8f73ed0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -56,6 +56,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeMetadata; import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.http.HttpPreRequest; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.http.netty4.Netty4HttpServerTransport; @@ -178,7 +179,6 @@ import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.RealmSettings; -import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; @@ -371,6 +371,7 @@ import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction; import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry; import org.elasticsearch.xpack.security.support.ExtensionComponents; +import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent; import org.elasticsearch.xpack.security.support.SecuritySystemIndices; import org.elasticsearch.xpack.security.transport.SecurityHttpSettings; import org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor; @@ -409,7 +410,6 @@ import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE; -import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.CLIENT_AUTHENTICATION_SHARED_SECRET; import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.INCLUDED_RESERVED_ROLES_SETTING; import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED; import static org.elasticsearch.xpack.security.transport.SSLEngineUtils.extractClientCertificates; @@ -563,6 +563,7 @@ public class Security extends Plugin private final SetOnce workflowService = new SetOnce<>(); private final SetOnce realms = new SetOnce<>(); private final SetOnce client = new SetOnce<>(); + private final SetOnce> reloadableComponents = new SetOnce<>(); public Security(Settings settings) { this(settings, Collections.emptyList()); @@ -634,8 +635,8 @@ protected Client getClient() { return client.get(); } - protected Realms getRealms() { - return realms.get(); + protected List getReloadableSecurityComponents() { + return this.reloadableComponents.get(); } @Override @@ -1045,6 +1046,13 @@ Collection createComponents( cacheInvalidatorRegistry.validate(); + this.reloadableComponents.set( + components.stream() + .filter(ReloadableSecurityComponent.class::isInstance) + .map(ReloadableSecurityComponent.class::cast) + .collect(Collectors.toUnmodifiableList()) + ); + return components; } @@ -1397,7 +1405,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (enabled == false) { return emptyList(); @@ -1946,11 +1955,13 @@ public void reload(Settings settings) throws Exception { reloadExceptions.add(ex); } - try { - reloadSharedSecretsForJwtRealms(settings); - } catch (Exception ex) { - reloadExceptions.add(ex); - } + this.getReloadableSecurityComponents().forEach(component -> { + try { + component.reload(settings); + } catch (Exception ex) { + reloadExceptions.add(ex); + } + }); if (false == reloadExceptions.isEmpty()) { final var combinedException = new ElasticsearchException( @@ -1964,16 +1975,6 @@ public void reload(Settings settings) throws Exception { } } - private void reloadSharedSecretsForJwtRealms(Settings settingsWithKeystore) { - getRealms().stream().filter(r -> JwtRealmSettings.TYPE.equals(r.realmRef().getType())).forEach(realm -> { - if (realm instanceof JwtRealm jwtRealm) { - jwtRealm.rotateClientSecret( - CLIENT_AUTHENTICATION_SHARED_SECRET.getConcreteSettingForNamespace(realm.realmRef().getName()).get(settingsWithKeystore) - ); - } - }); - } - /** * This method uses a transport action internally to access classes that are injectable but not part of the plugin contract. * See {@link TransportReloadRemoteClusterCredentialsAction} for more context. diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java index 0735eccff9913..2ca70bee55d4e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.common.Strings; @@ -35,6 +36,7 @@ import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; +import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent; import java.io.Closeable; import java.io.IOException; @@ -57,7 +59,7 @@ /** * Serves as a realms registry (also responsible for ordering the realms appropriately) */ -public class Realms extends AbstractLifecycleComponent implements Iterable { +public class Realms extends AbstractLifecycleComponent implements Iterable, ReloadableSecurityComponent { private static final Logger logger = LogManager.getLogger(Realms.class); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName()); @@ -566,4 +568,23 @@ private static Map convertToMapOfLists(Map map) } return converted; } + + @Override + public void reload(Settings settings) { + final List reloadExceptions = new ArrayList<>(); + for (Realm realm : this.allConfiguredRealms) { + if (realm instanceof ReloadableSecurityComponent reloadableRealm) { + try { + reloadableRealm.reload(settings); + } catch (Exception e) { + reloadExceptions.add(e); + } + } + } + if (false == reloadExceptions.isEmpty()) { + final var combinedException = new ElasticsearchException("secure settings reload failed for one or more realms"); + reloadExceptions.forEach(combinedException::addSuppressed); + throw combinedException; + } + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java index d8b0575c54d36..a541eef2f07f6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.settings.RotatableSecret; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -40,6 +41,7 @@ import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.support.ClaimParser; import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport; +import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent; import java.util.Collection; import java.util.Collections; @@ -51,13 +53,14 @@ import static java.lang.String.join; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.CLIENT_AUTHENTICATION_SHARED_SECRET; import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.CLIENT_AUTH_SHARED_SECRET_ROTATION_GRACE_PERIOD; /** * JWT realms supports JWTs as bearer tokens for authenticating to Elasticsearch. * For security, it is recommended to authenticate the client too. */ -public class JwtRealm extends Realm implements CachingRealm, Releasable { +public class JwtRealm extends Realm implements CachingRealm, ReloadableSecurityComponent, Releasable { private static final String LATEST_MALFORMED_JWT = "_latest_malformed_jwt"; @@ -399,8 +402,23 @@ public void usageStats(final ActionListener> listener) { }, listener::onFailure)); } - public void rotateClientSecret(SecureString clientSecret) { - this.clientAuthenticationSharedSecret.rotate(clientSecret, config.getSetting(CLIENT_AUTH_SHARED_SECRET_ROTATION_GRACE_PERIOD)); + @Override + public void reload(Settings settings) { + final SecureString newClientSharedSecret = CLIENT_AUTHENTICATION_SHARED_SECRET.getConcreteSettingForNamespace( + this.realmRef().getName() + ).get(settings); + + JwtUtil.validateClientAuthenticationSettings( + RealmSettings.getFullSettingKey(this.config, JwtRealmSettings.CLIENT_AUTHENTICATION_TYPE), + this.clientAuthenticationType, + RealmSettings.getFullSettingKey(this.config, JwtRealmSettings.CLIENT_AUTHENTICATION_SHARED_SECRET), + new RotatableSecret(newClientSharedSecret) + ); + + this.clientAuthenticationSharedSecret.rotate( + newClientSharedSecret, + config.getSetting(CLIENT_AUTH_SHARED_SECRET_ROTATION_GRACE_PERIOD) + ); } /** diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java index 6908519483c3e..495fdfe4cc0f2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; import static org.elasticsearch.xpack.security.authc.ldap.support.LdapUtils.attributesToSearchFor; import static org.elasticsearch.xpack.security.authc.ldap.support.LdapUtils.createFilter; @@ -102,7 +103,8 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory { groupResolver, metadataResolver, domainDN, - threadPool + threadPool, + this::getBindRequest ); downLevelADAuthenticator = new DownLevelADAuthenticator( config, @@ -117,7 +119,8 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory { ldapPort, ldapsPort, gcLdapPort, - gcLdapsPort + gcLdapsPort, + this::getBindRequest ); upnADAuthenticator = new UpnADAuthenticator( config, @@ -127,7 +130,8 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory { groupResolver, metadataResolver, domainDN, - threadPool + threadPool, + this::getBindRequest ); } @@ -187,7 +191,7 @@ void getUnauthenticatedSessionWithoutPool(String user, ActionListener bindRequestSupplier; final ThreadPool threadPool; ADAuthenticator( @@ -288,7 +291,8 @@ abstract static class ADAuthenticator { String domainDN, Setting.AffixSetting userSearchFilterSetting, String defaultUserSearchFilter, - ThreadPool threadPool + ThreadPool threadPool, + Supplier bindRequestSupplier ) { this.realm = realm; this.timeout = timeout; @@ -296,11 +300,7 @@ abstract static class ADAuthenticator { this.logger = logger; this.groupsResolver = groupsResolver; this.metadataResolver = metadataResolver; - this.bindDN = getBindDN(realm); - this.bindPassword = realm.getSetting( - PoolingSessionFactorySettings.SECURE_BIND_PASSWORD, - () -> realm.getSetting(PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD) - ); + this.bindRequestSupplier = bindRequestSupplier; this.threadPool = threadPool; userSearchDN = realm.getSetting(ActiveDirectorySessionFactorySettings.AD_USER_SEARCH_BASEDN_SETTING, () -> domainDN); userSearchScope = LdapSearchScope.resolve( @@ -348,10 +348,10 @@ protected void doRun() throws Exception { }, e -> { listener.onFailure(e); })); } }; - if (bindDN.isEmpty()) { + final SimpleBindRequest bind = bindRequestSupplier.get(); + if (bind.getBindDN().isEmpty()) { searchRunnable.run(); } else { - final SimpleBindRequest bind = new SimpleBindRequest(bindDN, CharArrays.toUtf8Bytes(bindPassword.getChars())); LdapUtils.maybeForkThenBind(connection, bind, true, threadPool, searchRunnable); } } @@ -423,7 +423,8 @@ static class DefaultADAuthenticator extends ADAuthenticator { GroupsResolver groupsResolver, LdapMetadataResolver metadataResolver, String domainDN, - ThreadPool threadPool + ThreadPool threadPool, + Supplier bindRequestSupplier ) { super( realm, @@ -435,7 +436,8 @@ static class DefaultADAuthenticator extends ADAuthenticator { domainDN, ActiveDirectorySessionFactorySettings.AD_USER_SEARCH_FILTER_SETTING, "(&(objectClass=user)(|(sAMAccountName={0})(userPrincipalName={0}@" + domainName(realm) + ")))", - threadPool + threadPool, + bindRequestSupplier ); domainName = domainName(realm); } @@ -503,7 +505,8 @@ static class DownLevelADAuthenticator extends ADAuthenticator { int ldapPort, int ldapsPort, int gcLdapPort, - int gcLdapsPort + int gcLdapsPort, + Supplier bindRequestSupplier ) { super( config, @@ -515,7 +518,8 @@ static class DownLevelADAuthenticator extends ADAuthenticator { domainDN, ActiveDirectorySessionFactorySettings.AD_DOWN_LEVEL_USER_SEARCH_FILTER_SETTING, DOWN_LEVEL_FILTER, - threadPool + threadPool, + bindRequestSupplier ); this.domainDN = domainDN; this.sslService = sslService; @@ -605,10 +609,9 @@ void netBiosDomainNameToDn( ) ); final byte[] passwordBytes = CharArrays.toUtf8Bytes(password.getChars()); - final boolean bindAsAuthenticatingUser = this.bindDN.isEmpty(); - final SimpleBindRequest bind = bindAsAuthenticatingUser - ? new SimpleBindRequest(username, passwordBytes) - : new SimpleBindRequest(bindDN, CharArrays.toUtf8Bytes(bindPassword.getChars())); + final SimpleBindRequest bindRequest = bindRequestSupplier.get(); + final boolean bindAsAuthenticatingUser = bindRequest.getBindDN().isEmpty(); + final SimpleBindRequest bind = bindAsAuthenticatingUser ? new SimpleBindRequest(username, passwordBytes) : bindRequest; ActionRunnable body = new ActionRunnable<>(listener) { @Override protected void doRun() throws Exception { @@ -705,7 +708,8 @@ static class UpnADAuthenticator extends ADAuthenticator { GroupsResolver groupsResolver, LdapMetadataResolver metadataResolver, String domainDN, - ThreadPool threadPool + ThreadPool threadPool, + Supplier bindRequestSupplier ) { super( config, @@ -717,7 +721,8 @@ static class UpnADAuthenticator extends ADAuthenticator { domainDN, ActiveDirectorySessionFactorySettings.AD_UPN_USER_SEARCH_FILTER_SETTING, UPN_USER_FILTER, - threadPool + threadPool, + bindRequestSupplier ); if (userSearchFilter.contains("{0}")) { deprecationLogger.warn( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java index 91b49f39b4b3c..48dd0fda5b569 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -40,6 +41,7 @@ import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport; import org.elasticsearch.xpack.security.authc.support.mapper.CompositeRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; +import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent; import java.util.HashMap; import java.util.List; @@ -57,7 +59,7 @@ /** * Authenticates username/password tokens against ldap, locates groups and maps them to roles. */ -public final class LdapRealm extends CachingUsernamePasswordRealm { +public final class LdapRealm extends CachingUsernamePasswordRealm implements ReloadableSecurityComponent { private final SessionFactory sessionFactory; private final UserRoleMapper roleMapper; @@ -217,6 +219,11 @@ public void usageStats(ActionListener> listener) { }, listener::onFailure)); } + @Override + public void reload(Settings settings) { + this.sessionFactory.reload(settings); + } + private static void buildUser( LdapSession session, String username, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java index 4e390c86ba1f1..e0c57dc8b19a3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactory.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.CharArrays; import org.elasticsearch.core.IOUtils; @@ -120,6 +121,11 @@ void loop() { } } + @Override + public void reload(Settings settings) { + // nothing to reload in DN template mode + } + /** * Securely escapes the username and inserts it into the template using MessageFormat * diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java index d177ffbefebf5..362891ae9db7f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactory.java @@ -127,6 +127,7 @@ void getSessionWithPool(LDAPConnectionPool connectionPool, String user, SecureSt void getSessionWithoutPool(String user, SecureString password, ActionListener listener) { try { final LDAPConnection connection = LdapUtils.privilegedConnect(serverSet::getConnection); + final SimpleBindRequest bindCredentials = this.getBindRequest(); LdapUtils.maybeForkThenBind(connection, bindCredentials, true, threadPool, new AbstractRunnable() { @Override protected void doRun() throws Exception { @@ -222,7 +223,7 @@ void getUnauthenticatedSessionWithPool(LDAPConnectionPool connectionPool, String void getUnauthenticatedSessionWithoutPool(String user, ActionListener listener) { try { final LDAPConnection connection = LdapUtils.privilegedConnect(serverSet::getConnection); - LdapUtils.maybeForkThenBind(connection, bindCredentials, true, threadPool, new AbstractRunnable() { + LdapUtils.maybeForkThenBind(connection, getBindRequest(), true, threadPool, new AbstractRunnable() { @Override protected void doRun() throws Exception { findUser(user, connection, ActionListener.wrap((entry) -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java index ae48b1c1dc1b2..24bdb9243aef7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/PoolingSessionFactory.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CharArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -32,6 +33,7 @@ import org.elasticsearch.xpack.security.authc.ldap.support.LdapUtils; import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import static org.elasticsearch.core.Strings.format; @@ -46,7 +48,8 @@ abstract class PoolingSessionFactory extends SessionFactory implements Releasabl private final boolean useConnectionPool; private final LDAPConnectionPool connectionPool; - final SimpleBindRequest bindCredentials; + private final String bindDn; + private final AtomicReference bindRequest; final LdapSession.GroupsResolver groupResolver; /** @@ -69,27 +72,36 @@ abstract class PoolingSessionFactory extends SessionFactory implements Releasabl ) throws LDAPException { super(config, sslService, threadPool); this.groupResolver = groupResolver; + this.bindDn = bindDn; + this.bindRequest = new AtomicReference<>(buildBindRequest(config.settings())); + this.useConnectionPool = config.getSetting(poolingEnabled); + if (useConnectionPool) { + this.connectionPool = createConnectionPool(config, serverSet, timeout, logger, bindRequest.get(), healthCheckDNSupplier); + } else { + this.connectionPool = null; + } + } + private SimpleBindRequest buildBindRequest(Settings settings) { final byte[] bindPassword; - if (config.hasSetting(LEGACY_BIND_PASSWORD)) { - if (config.hasSetting(SECURE_BIND_PASSWORD)) { + final Setting legacyPasswordSetting = config.getConcreteSetting(LEGACY_BIND_PASSWORD); + final Setting securePasswordSetting = config.getConcreteSetting(SECURE_BIND_PASSWORD); + + if (legacyPasswordSetting.exists(settings)) { + if (securePasswordSetting.exists(settings)) { throw new IllegalArgumentException( - "You cannot specify both [" - + RealmSettings.getFullSettingKey(config, LEGACY_BIND_PASSWORD) - + "] and [" - + RealmSettings.getFullSettingKey(config, SECURE_BIND_PASSWORD) - + "]" + "You cannot specify both [" + legacyPasswordSetting.getKey() + "] and [" + securePasswordSetting.getKey() + "]" ); } - bindPassword = CharArrays.toUtf8Bytes(config.getSetting(LEGACY_BIND_PASSWORD).getChars()); - } else if (config.hasSetting(SECURE_BIND_PASSWORD)) { - bindPassword = CharArrays.toUtf8Bytes(config.getSetting(SECURE_BIND_PASSWORD).getChars()); + bindPassword = CharArrays.toUtf8Bytes(legacyPasswordSetting.get(settings).getChars()); + } else if (securePasswordSetting.exists(settings)) { + bindPassword = CharArrays.toUtf8Bytes(securePasswordSetting.get(settings).getChars()); } else { bindPassword = null; } - if (bindDn == null) { - bindCredentials = new SimpleBindRequest(); + if (this.bindDn == null) { + return new SimpleBindRequest(); } else { if (bindPassword == null) { deprecationLogger.critical( @@ -104,17 +116,32 @@ abstract class PoolingSessionFactory extends SessionFactory implements Releasabl RealmSettings.getFullSettingKey(config, LEGACY_BIND_PASSWORD) ); } - bindCredentials = new SimpleBindRequest(bindDn, bindPassword); + return new SimpleBindRequest(this.bindDn, bindPassword); } + } - this.useConnectionPool = config.getSetting(poolingEnabled); - if (useConnectionPool) { - this.connectionPool = createConnectionPool(config, serverSet, timeout, logger, bindCredentials, healthCheckDNSupplier); - } else { - this.connectionPool = null; + @Override + public void reload(Settings settings) { + final SimpleBindRequest oldRequest = bindRequest.get(); + final SimpleBindRequest newRequest = buildBindRequest(settings); + if (bindRequestEquals(newRequest, oldRequest) == false) { + if (bindRequest.compareAndSet(oldRequest, newRequest)) { + if (connectionPool != null) { + // When a connection is open and already bound, changing the bind password does not affect + // the existing pooled connections. LDAP connections are stateful, and once a connection is + // established and bound, it remains open until explicitly closed or until a connection + // timeout occurs. Changing the bind password on the LDAP server does not automatically + // invalidate existing connections. Hence, simply setting the new bind request is sufficient. + connectionPool.setBindRequest(bindRequest.get()); + } + } } } + private static boolean bindRequestEquals(SimpleBindRequest req1, SimpleBindRequest req2) { + return req1.getBindDN().contentEquals(req2.getBindDN()) && req1.getPassword().equalsIgnoreType(req2.getPassword()); + } + @Override public final void session(String user, SecureString password, ActionListener listener) { if (useConnectionPool) { @@ -238,4 +265,8 @@ LDAPConnectionPool getConnectionPool() { return connectionPool; } + SimpleBindRequest getBindRequest() { + return bindRequest.get(); + } + } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java index 5d260266d3f20..2a8625b2d93fb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactory.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySettings; import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import org.elasticsearch.xpack.core.ssl.SSLService; +import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent; import java.io.Closeable; import java.io.IOException; @@ -49,7 +50,7 @@ * } * */ -public abstract class SessionFactory implements Closeable { +public abstract class SessionFactory implements Closeable, ReloadableSecurityComponent { private static final Pattern STARTS_WITH_LDAPS = Pattern.compile("^ldaps:.*", Pattern.CASE_INSENSITIVE); private static final Pattern STARTS_WITH_LDAP = Pattern.compile("^ldap:.*", Pattern.CASE_INSENSITIVE); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ReloadableSecurityComponent.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ReloadableSecurityComponent.java new file mode 100644 index 0000000000000..7bd9023964715 --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ReloadableSecurityComponent.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security.support; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.ReloadablePlugin; + +/** + * This interface allows adding support for reload operations (on secure settings change) in a generic way for security components. + * The implementors of this interface will be called when the values of {@code SecureSetting}s should be reloaded by security plugin. + * For more information about reloading plugin secure settings, see {@link ReloadablePlugin}. + */ +public interface ReloadableSecurityComponent { + + /** + * Called when a reload security settings action is executed. The reload operation + * must be completed when this method returns. Strictly speaking, the + * settings argument should not be accessed outside of this method's + * call stack, as any values stored in the node's keystore (see {@code SecureSetting}) + * will not otherwise be retrievable. + *

    + * There is no guarantee that the secure setting's values have actually changed. + * Hence, it's up to implementor to detect if the actual internal reloading is + * necessary. + *

    + * Any failure during the reloading should be signaled by raising an exception. + *

    + * For additional info, see also: {@link ReloadablePlugin#reload(Settings)}. + * + * @param settings + * Settings include the initial node's settings and all decrypted + * secure settings from the keystore. Absence of a particular secure + * setting may mean that the setting was either never configured or + * that it was simply removed. + */ + void reload(Settings settings); + +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 6cd12858a12c1..5cffc048d9416 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -97,11 +97,13 @@ import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; +import org.elasticsearch.xpack.security.authc.jwt.JwtRealm; import org.elasticsearch.xpack.security.authc.service.CachingServiceAccountTokenStore; import org.elasticsearch.xpack.security.operator.DefaultOperatorOnlyRegistry; import org.elasticsearch.xpack.security.operator.OperatorOnlyRegistry; import org.elasticsearch.xpack.security.operator.OperatorPrivileges; import org.elasticsearch.xpack.security.operator.OperatorPrivilegesViolation; +import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent; import org.hamcrest.Matchers; import org.junit.After; @@ -121,7 +123,6 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static org.elasticsearch.test.LambdaMatchers.falseWith; @@ -142,7 +143,9 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -967,8 +970,8 @@ public void testReload() throws Exception { final PlainActionFuture value = new PlainActionFuture<>(); final Client mockedClient = mock(Client.class); - final Realms mockedRealms = mock(Realms.class); - when(mockedRealms.stream()).thenReturn(Stream.of()); + final JwtRealm mockedJwtRealm = mock(JwtRealm.class); + final List reloadableComponents = List.of(mockedJwtRealm); doAnswer((inv) -> { @SuppressWarnings("unchecked") @@ -984,8 +987,8 @@ protected Client getClient() { } @Override - protected Realms getRealms() { - return mockedRealms; + protected List getReloadableSecurityComponents() { + return reloadableComponents; } }; @@ -993,14 +996,16 @@ protected Realms getRealms() { security.reload(inputSettings); verify(mockedClient).execute(eq(ActionTypes.RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION), any(), any()); - verify(mockedRealms).stream(); + verify(mockedJwtRealm).reload(same(inputSettings)); } - public void testReloadWithFailures() { + public void testReloadWithFailures() throws Exception { final Settings settings = Settings.builder().put("xpack.security.enabled", true).put("path.home", createTempDir()).build(); final boolean failRemoteClusterCredentialsReload = randomBoolean(); final Client mockedClient = mock(Client.class); + final JwtRealm mockedJwtRealm = mock(JwtRealm.class); + final List reloadableComponents = List.of(mockedJwtRealm); if (failRemoteClusterCredentialsReload) { doAnswer((inv) -> { @SuppressWarnings("unchecked") @@ -1017,12 +1022,9 @@ public void testReloadWithFailures() { }).when(mockedClient).execute(eq(ActionTypes.RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION), any(), any()); } - final Realms mockedRealms = mock(Realms.class); final boolean failRealmsReload = (false == failRemoteClusterCredentialsReload) || randomBoolean(); if (failRealmsReload) { - when(mockedRealms.stream()).thenThrow(new RuntimeException("failed jwt realms reload")); - } else { - when(mockedRealms.stream()).thenReturn(Stream.of()); + doThrow(new RuntimeException("failed jwt realms reload")).when(mockedJwtRealm).reload(any()); } security = new Security(settings, Collections.emptyList()) { @Override @@ -1031,8 +1033,8 @@ protected Client getClient() { } @Override - protected Realms getRealms() { - return mockedRealms; + protected List getReloadableSecurityComponents() { + return reloadableComponents; } }; @@ -1050,7 +1052,7 @@ protected Realms getRealms() { } // Verify both called despite failure verify(mockedClient).execute(eq(ActionTypes.RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION), any(), any()); - verify(mockedRealms).stream(); + verify(mockedJwtRealm).reload(same(inputSettings)); } public void testLoadNoExtensions() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index e9af65bd8fc4a..2fb8a69ec9601 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -589,6 +589,44 @@ public void testMandatorySettings() throws Exception { ); } + public void testReloadBindPassword() throws Exception { + final RealmConfig.RealmIdentifier realmIdentifier = realmId("testReloadBindPassword"); + final boolean useLegacyBindPassword = randomBoolean(); + final boolean pooled = randomBoolean(); + final Settings.Builder builder = Settings.builder() + .put(getFullSettingKey(realmIdentifier, PoolingSessionFactorySettings.BIND_DN), "CN=ironman@ad.test.elasticsearch.com") + .put(getFullSettingKey(realmIdentifier.getName(), ActiveDirectorySessionFactorySettings.POOL_ENABLED), pooled) + // explicitly disabling cache to always authenticate against AD server + .put(getFullSettingKey(realmIdentifier, CachingUsernamePasswordRealmSettings.CACHE_TTL_SETTING), -1) + // due to limitations of AD server we cannot change BIND password dynamically, + // so we start with the wrong (random) password and then reload and check if authentication succeeds + .put(bindPasswordSettings(realmIdentifier, useLegacyBindPassword, randomAlphaOfLengthBetween(3, 7))); + + Settings settings = settings(realmIdentifier, builder.build()); + RealmConfig config = setupRealm(realmIdentifier, settings); + try (ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, sslService, threadPool)) { + DnRoleMapper roleMapper = new DnRoleMapper(config, resourceWatcherService); + LdapRealm realm = new LdapRealm(config, sessionFactory, roleMapper, threadPool); + realm.initialize(Collections.singleton(realm), licenseState); + + { + final PlainActionFuture> future = new PlainActionFuture<>(); + realm.authenticate(new UsernamePasswordToken("CN=Thor", new SecureString(PASSWORD.toCharArray())), future); + final AuthenticationResult result = future.actionGet(); + assertThat(result.toString(), result.getStatus(), is(AuthenticationResult.Status.CONTINUE)); + } + + realm.reload(bindPasswordSettings(realmIdentifier, useLegacyBindPassword, PASSWORD)); + + { + final PlainActionFuture> future = new PlainActionFuture<>(); + realm.authenticate(new UsernamePasswordToken("CN=Thor", new SecureString(PASSWORD.toCharArray())), future); + final AuthenticationResult result = future.actionGet(); + assertThat(result.toString(), result.getStatus(), is(AuthenticationResult.Status.SUCCESS)); + } + } + } + private void assertSingleLdapServer(ActiveDirectorySessionFactory sessionFactory, String hostname, int port) { assertThat(sessionFactory.getServerSet(), instanceOf(FailoverServerSet.class)); FailoverServerSet fss = (FailoverServerSet) sessionFactory.getServerSet(); @@ -628,4 +666,17 @@ private User getAndVerifyAuthUser(PlainActionFuture> assertThat(user, is(notNullValue())); return user; } + + private Settings bindPasswordSettings(RealmConfig.RealmIdentifier realmIdentifier, boolean useLegacyBindPassword, String password) { + if (useLegacyBindPassword) { + return Settings.builder() + .put(getFullSettingKey(realmIdentifier, PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD), password) + .build(); + } else { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString(getFullSettingKey(realmIdentifier, PoolingSessionFactorySettings.SECURE_BIND_PASSWORD), password); + return Settings.builder().setSecureSettings(secureSettings).build(); + } + } + } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmReloadTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmReloadTests.java new file mode 100644 index 0000000000000..cf62b8355644b --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmReloadTests.java @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.security.authc.ldap; + +import com.unboundid.ldap.sdk.LDAPException; +import com.unboundid.ldap.sdk.LDAPResult; +import com.unboundid.ldap.sdk.Modification; +import com.unboundid.ldap.sdk.ModificationType; +import com.unboundid.ldap.sdk.ResultCode; + +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.SecureSettings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.ssl.SslVerificationMode; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.license.MockLicenseState; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; +import org.elasticsearch.xpack.core.security.authc.RealmConfig; +import org.elasticsearch.xpack.core.security.authc.RealmSettings; +import org.elasticsearch.xpack.core.security.authc.ldap.LdapUserSearchSessionFactorySettings; +import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings; +import org.elasticsearch.xpack.core.security.authc.ldap.SearchGroupsResolverSettings; +import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope; +import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings; +import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; +import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.core.ssl.SSLService; +import org.elasticsearch.xpack.security.Security; +import org.elasticsearch.xpack.security.authc.ldap.support.LdapTestCase; +import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; +import org.junit.After; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.function.Function; + +import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey; +import static org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySettings.URLS_SETTING; +import static org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings.VERIFICATION_MODE_SETTING_REALM; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class LdapRealmReloadTests extends LdapTestCase { + + public static final String BIND_DN = "cn=Thomas Masterman Hardy,ou=people,o=sevenSeas"; + public static final String INITIAL_BIND_PASSWORD = "pass"; + public static final UsernamePasswordToken LDAP_USER_AUTH_TOKEN = new UsernamePasswordToken( + "jsamuel@royalnavy.mod.uk", + new SecureString("pass".toCharArray()) + ); + + private static final Settings defaultRealmSettings = Settings.builder() + .put(getFullSettingKey(REALM_IDENTIFIER.getName(), LdapUserSearchSessionFactorySettings.SEARCH_BASE_DN), "") + .put(getFullSettingKey(REALM_IDENTIFIER, PoolingSessionFactorySettings.BIND_DN), BIND_DN) + .put(getFullSettingKey(REALM_IDENTIFIER, SearchGroupsResolverSettings.SCOPE), LdapSearchScope.SUB_TREE) + .put(getFullSettingKey(REALM_IDENTIFIER, VERIFICATION_MODE_SETTING_REALM), SslVerificationMode.CERTIFICATE) + // explicitly disabling cache to always authenticate against LDAP server + .put(getFullSettingKey(REALM_IDENTIFIER, CachingUsernamePasswordRealmSettings.CACHE_TTL_SETTING), -1) + .put(getFullSettingKey(REALM_IDENTIFIER, RealmSettings.ORDER_SETTING), 0) + .build(); + + private ResourceWatcherService resourceWatcherService; + private Settings defaultGlobalSettings; + private MockLicenseState licenseState; + private ThreadPool threadPool; + + @Before + public void init() throws Exception { + licenseState = mock(MockLicenseState.class); + threadPool = new TestThreadPool("ldap realm reload tests"); + resourceWatcherService = new ResourceWatcherService(Settings.EMPTY, threadPool); + defaultGlobalSettings = Settings.builder().put("path.home", createTempDir()).build(); + when(licenseState.isAllowed(Security.DELEGATED_AUTHORIZATION_FEATURE)).thenReturn(true); + } + + @After + public void shutdown() throws InterruptedException { + resourceWatcherService.close(); + terminate(threadPool); + } + + private RealmConfig getRealmConfig(RealmConfig.RealmIdentifier identifier, Settings settings) { + final Environment env = TestEnvironment.newEnvironment(settings); + return new RealmConfig(identifier, settings, env, new ThreadContext(settings)); + } + + public void testReloadWithoutConnectionPool() throws Exception { + final boolean useLegacyBindSetting = randomBoolean(); + final Settings bindPasswordSettings; + if (useLegacyBindSetting) { + bindPasswordSettings = Settings.builder() + .put(getFullSettingKey(REALM_IDENTIFIER, PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD), INITIAL_BIND_PASSWORD) + .build(); + } else { + bindPasswordSettings = Settings.builder() + .setSecureSettings(secureSettings(PoolingSessionFactorySettings.SECURE_BIND_PASSWORD, INITIAL_BIND_PASSWORD)) + .build(); + } + final Settings settings = Settings.builder() + .put(getFullSettingKey(REALM_IDENTIFIER.getName(), LdapUserSearchSessionFactorySettings.POOL_ENABLED), false) + .putList(getFullSettingKey(REALM_IDENTIFIER, URLS_SETTING), ldapUrls()) + .put(defaultRealmSettings) + .put(defaultGlobalSettings) + .put(bindPasswordSettings) + .build(); + final RealmConfig config = getRealmConfig(REALM_IDENTIFIER, settings); + try (SessionFactory sessionFactory = LdapRealm.sessionFactory(config, new SSLService(config.env()), threadPool)) { + assertThat(sessionFactory, is(instanceOf(LdapUserSearchSessionFactory.class))); + + LdapRealm ldap = new LdapRealm(config, sessionFactory, buildGroupAsRoleMapper(resourceWatcherService), threadPool); + ldap.initialize(Collections.singleton(ldap), licenseState); + + // Verify authentication is successful before the password change + authenticateUserAndAssertStatus(ldap, AuthenticationResult.Status.SUCCESS); + + // Generate new password and reload only on ES side + final String newBindPassword = randomAlphaOfLengthBetween(5, 10); + final Settings updatedBindPasswordSettings; + if (useLegacyBindSetting) { + updatedBindPasswordSettings = Settings.builder() + .put(getFullSettingKey(REALM_IDENTIFIER, PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD), newBindPassword) + .build(); + } else { + updatedBindPasswordSettings = Settings.builder() + .setSecureSettings(secureSettings(PoolingSessionFactorySettings.SECURE_BIND_PASSWORD, newBindPassword)) + .build(); + } + ldap.reload(updatedBindPasswordSettings); + authenticateUserAndAssertStatus(ldap, AuthenticationResult.Status.CONTINUE); + + // Change password on LDAP server side and check that authentication works + changeUserPasswordOnLdapServers(BIND_DN, newBindPassword); + authenticateUserAndAssertStatus(ldap, AuthenticationResult.Status.SUCCESS); + + if (useLegacyBindSetting) { + assertSettingDeprecationsAndWarnings( + new Setting[] { + PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD.apply(REALM_IDENTIFIER.getType()) + .getConcreteSettingForNamespace(REALM_IDENTIFIER.getName()) } + ); + } + } + } + + public void testReloadWithConnectionPool() throws Exception { + final boolean useLegacyBindSetting = randomBoolean(); + final Settings bindPasswordSettings; + if (useLegacyBindSetting) { + bindPasswordSettings = Settings.builder() + .put(getFullSettingKey(REALM_IDENTIFIER, PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD), INITIAL_BIND_PASSWORD) + .build(); + } else { + bindPasswordSettings = Settings.builder() + .setSecureSettings(secureSettings(PoolingSessionFactorySettings.SECURE_BIND_PASSWORD, INITIAL_BIND_PASSWORD)) + .build(); + } + final Settings settings = Settings.builder() + .put(getFullSettingKey(REALM_IDENTIFIER.getName(), LdapUserSearchSessionFactorySettings.POOL_ENABLED), true) + .putList(getFullSettingKey(REALM_IDENTIFIER, URLS_SETTING), ldapUrls()) + .put(defaultRealmSettings) + .put(defaultGlobalSettings) + .put(bindPasswordSettings) + .build(); + final RealmConfig config = getRealmConfig(REALM_IDENTIFIER, settings); + try (SessionFactory sessionFactory = LdapRealm.sessionFactory(config, new SSLService(config.env()), threadPool)) { + assertThat(sessionFactory, is(instanceOf(LdapUserSearchSessionFactory.class))); + + LdapRealm ldap = new LdapRealm(config, sessionFactory, buildGroupAsRoleMapper(resourceWatcherService), threadPool); + ldap.initialize(Collections.singleton(ldap), licenseState); + + // When a connection is open and already bound, changing the bind password generally + // does not affect the existing pooled connection. LDAP connections are stateful, + // and once a connection is established and bound, it remains open until explicitly closed + // or until a connection timeout occurs. Changing the bind password on the server + // does not automatically invalidate existing connections. Hence, we are skipping + // here the check that the authentication works before re-loading bind password, + // since this check would create and bind a new connection using old password. + + // Generate a new password and reload only on ES side + final String newBindPassword = randomAlphaOfLengthBetween(5, 10); + final Settings updatedBindPasswordSettings; + if (useLegacyBindSetting) { + updatedBindPasswordSettings = Settings.builder() + .put(getFullSettingKey(REALM_IDENTIFIER, PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD), newBindPassword) + .build(); + } else { + updatedBindPasswordSettings = Settings.builder() + .setSecureSettings(secureSettings(PoolingSessionFactorySettings.SECURE_BIND_PASSWORD, newBindPassword)) + .build(); + } + ldap.reload(updatedBindPasswordSettings); + // Using new bind password should fail since we did not update it on LDAP server side. + authenticateUserAndAssertStatus(ldap, AuthenticationResult.Status.CONTINUE); + + // Change password on LDAP server side and check that authentication works now. + changeUserPasswordOnLdapServers(BIND_DN, newBindPassword); + authenticateUserAndAssertStatus(ldap, AuthenticationResult.Status.SUCCESS); + + if (useLegacyBindSetting) { + assertSettingDeprecationsAndWarnings( + new Setting[] { + PoolingSessionFactorySettings.LEGACY_BIND_PASSWORD.apply(REALM_IDENTIFIER.getType()) + .getConcreteSettingForNamespace(REALM_IDENTIFIER.getName()) } + ); + } + } + } + + private void authenticateUserAndAssertStatus(LdapRealm ldap, AuthenticationResult.Status expectedAuthStatus) { + final PlainActionFuture> future = new PlainActionFuture<>(); + ldap.authenticate(LDAP_USER_AUTH_TOKEN, future); + final AuthenticationResult result = future.actionGet(); + assertThat(result.getStatus(), is(expectedAuthStatus)); + } + + private void changeUserPasswordOnLdapServers(String userDn, String newPassword) { + Arrays.stream(ldapServers).forEach(ldapServer -> { + ldapServer.getPasswordAttributes().forEach(passwordAttribute -> { + try { + LDAPResult result = ldapServer.modify(userDn, new Modification(ModificationType.REPLACE, "userPassword", newPassword)); + assertThat(result.getResultCode(), equalTo(ResultCode.SUCCESS)); + } catch (LDAPException e) { + fail(e, "failed to change " + passwordAttribute + " for user: " + userDn); + } + }); + }); + } + + private static SecureSettings secureSettings(Function> settingFactory, String value) { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString(getFullSettingKey(REALM_IDENTIFIER, settingFactory), value); + return secureSettings; + } +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java index 4daaee30e098d..acb4359b37323 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java @@ -146,14 +146,14 @@ public void testUserSearchSubTree() throws Exception { try { // auth try (LdapSession ldap = session(sessionFactory, user, userPass)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString(user)); } // lookup try (LdapSession ldap = unauthenticatedSession(sessionFactory, user)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString(user)); } @@ -291,14 +291,14 @@ public void testUserSearchBaseScopePassesWithCorrectBaseDN() throws Exception { try { // auth try (LdapSession ldap = session(sessionFactory, user, userPass)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString(user)); } // lookup try (LdapSession ldap = unauthenticatedSession(sessionFactory, user)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString(user)); } @@ -378,14 +378,14 @@ public void testUserSearchOneLevelScopePassesWithCorrectBaseDN() throws Exceptio try { // auth try (LdapSession ldap = session(sessionFactory, user, userPass)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString(user)); } // lookup try (LdapSession ldap = unauthenticatedSession(sessionFactory, user)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString(user)); } @@ -451,14 +451,14 @@ public void testUserSearchWithoutAttributePasses() throws Exception { try { // auth try (LdapSession ldap = session(sessionFactory, user, userPass)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString("William Bush")); } // lookup try (LdapSession ldap = unauthenticatedSession(sessionFactory, user)) { - assertConnectionValid(ldap.getConnection(), sessionFactory.bindCredentials); + assertConnectionValid(ldap.getConnection(), sessionFactory.getBindRequest()); String dn = ldap.userDn(); assertThat(dn, containsString("William Bush")); } @@ -600,8 +600,8 @@ public void testEmptyBindDNReturnsAnonymousBindRequest() throws LDAPException { new ThreadContext(globalSettings) ); try (LdapUserSearchSessionFactory searchSessionFactory = getLdapUserSearchSessionFactory(config, sslService, threadPool)) { - assertThat(searchSessionFactory.bindCredentials, notNullValue()); - assertThat(searchSessionFactory.bindCredentials.getBindDN(), is(emptyString())); + assertThat(searchSessionFactory.getBindRequest(), notNullValue()); + assertThat(searchSessionFactory.getBindRequest().getBindDN(), is(emptyString())); } assertDeprecationWarnings(config.identifier(), false, useLegacyBindPassword); } @@ -622,8 +622,8 @@ public void testThatBindRequestReturnsSimpleBindRequest() throws LDAPException { new ThreadContext(globalSettings) ); try (LdapUserSearchSessionFactory searchSessionFactory = getLdapUserSearchSessionFactory(config, sslService, threadPool)) { - assertThat(searchSessionFactory.bindCredentials, notNullValue()); - assertThat(searchSessionFactory.bindCredentials.getBindDN(), is("cn=ironman")); + assertThat(searchSessionFactory.getBindRequest(), notNullValue()); + assertThat(searchSessionFactory.getBindRequest().getBindDN(), is("cn=ironman")); } assertDeprecationWarnings(config.identifier(), false, useLegacyBindPassword); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java index a74b3bd426c75..466d0e3428d50 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java @@ -471,5 +471,10 @@ protected TestSessionFactory(RealmConfig config, SSLService sslService, ThreadPo public void session(String user, SecureString password, ActionListener listener) { listener.onResponse(null); } + + @Override + public void reload(Settings settings) { + // no-op + } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java index a49070786bb0e..e8804653d365e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryTests.java @@ -225,6 +225,11 @@ private SessionFactory createSessionFactory() { public void session(String user, SecureString password, ActionListener listener) { listener.onResponse(null); } + + @Override + public void reload(Settings settings) { + // no-op + } }; } } diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java index 234a77154a641..25c6f431e57c8 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -25,6 +26,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class ShutdownPlugin extends Plugin implements ActionPlugin { @@ -62,7 +64,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList(new RestPutShutdownNodeAction(), new RestDeleteShutdownNodeAction(), new RestGetShutdownStatusAction()); } diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java index 946d9c081658a..cb0344dd70ad5 100644 --- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java +++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; @@ -77,6 +78,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; @@ -180,7 +182,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { List handlers = new ArrayList<>(); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java index 9bbb08e89166e..2013a8ff53301 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java @@ -185,7 +185,6 @@ public void testNothingScheduledWhenNotRunning() throws InterruptedException { * Test new policies getting scheduled correctly, updated policies also being scheduled, * and deleted policies having their schedules cancelled. */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/44997") public void testPolicyCRUD() throws Exception { ClockMock clock = new ClockMock(); final AtomicInteger triggerCount = new AtomicInteger(0); @@ -280,7 +279,7 @@ public void testPolicyCRUD() throws Exception { clock.fastForwardSeconds(2); // The existing job should be cancelled and no longer trigger - assertThat(triggerCount.get(), equalTo(currentCount2)); + assertBusy(() -> assertThat(triggerCount.get(), equalTo(currentCount2))); assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.emptySet())); // When the service is no longer master, all jobs should be automatically cancelled diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java index fd8970f327ce9..124174a2a025b 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -25,6 +26,7 @@ import java.io.IOException; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class SnapshotRepositoryTestKit extends Plugin implements ActionPlugin { @@ -43,7 +45,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestRepositoryAnalyzeAction()); } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index b79bc11c36a2b..0b76e786b26be 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -38,11 +38,13 @@ import org.elasticsearch.geometry.simplify.SimplificationErrorCalculator; import org.elasticsearch.geometry.simplify.StreamingGeometrySimplifier; import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.plugins.SearchPlugin; @@ -63,7 +65,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -96,6 +97,7 @@ public void testMixedMissingValues() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .subAggregation(lineAggregationBuilder); long lonLat = (((long) GeoEncodingUtils.encodeLongitude(90.0)) << 32) | GeoEncodingUtils.encodeLatitude(45.0) & 0xffffffffL; @@ -146,6 +148,7 @@ public void testMissingGeoPointValueField() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .subAggregation(lineAggregationBuilder); // input @@ -177,6 +180,7 @@ public void testMissingSortField() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .subAggregation(lineAggregationBuilder); testCase(aggregationBuilder, iw -> { @@ -317,6 +321,7 @@ public void testOnePoint() throws IOException { .sort(sortConfig) .size(size); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .subAggregation(lineAggregationBuilder); double lon = GeoEncodingUtils.decodeLongitude(randomInt()); double lat = GeoEncodingUtils.decodeLatitude(randomInt()); @@ -800,7 +805,7 @@ private void assertGeoLine_TSDB( ArrayList fields = new ArrayList<>( Arrays.asList( new SortedDocValuesField("group_id", new BytesRef(testData.groups[g])), - new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.build().toBytesRef()) + new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.buildTsidHash().toBytesRef()) ) ); GeoPoint point = points.get(i); @@ -947,7 +952,13 @@ private void testCase( fieldTypes.add(new GeoPointFieldMapper.GeoPointFieldType("value_field")); } fieldTypes.add(new DateFieldMapper.DateFieldType("time_field")); - fieldTypes.add(new KeywordFieldMapper.KeywordFieldType("group_id", false, true, Collections.emptyMap())); + fieldTypes.add( + new KeywordFieldMapper.Builder("group_id", IndexVersion.current()).dimension(true) + .docValues(true) + .indexed(false) + .build(MapperBuilderContext.root(true, true)) + .fieldType() + ); fieldTypes.add(new NumberFieldMapper.NumberFieldType("sort_field", NumberFieldMapper.NumberType.LONG)); AggTestConfig aggTestConfig = new AggTestConfig(aggregationBuilder, fieldTypes.toArray(new MappedFieldType[0])); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java index 52a62f4b21d76..84c5825b89e83 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.LicensedFeature; @@ -41,6 +42,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class SqlPlugin extends Plugin implements ActionPlugin { @@ -115,7 +117,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList( diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java index 6513d72eaf1f8..47a76b4a5291d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java @@ -55,7 +55,8 @@ public void testSqlDisabledIsNoOp() { IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SettingsFilter(Collections.emptyList()), mock(IndexNameExpressionResolver.class), - () -> mock(DiscoveryNodes.class) + () -> mock(DiscoveryNodes.class), + null ), hasSize(7) ); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml index ef34e64ad41d7..099d8ad9171eb 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml @@ -61,10 +61,11 @@ setup: - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' --- -aggretate multi_terms: +aggregate multi_terms: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 + - do: search: size: 0 @@ -78,13 +79,39 @@ aggretate multi_terms: - field: k8s.pod.ip - length: { aggregations.m_terms.buckets: 3 } - - match: { aggregations.m_terms.buckets.0.key_as_string: "{k8s.pod.uid=df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset=pod}|10.10.55.3" } + - match: { aggregations.m_terms.buckets.0.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ|10.10.55.3" } - match: { aggregations.m_terms.buckets.0.doc_count: 4 } - - match: { aggregations.m_terms.buckets.1.key_as_string: "{k8s.pod.uid=947e4ced-1786-4e53-9e0c-5c447e959507, metricset=pod}|10.10.55.1" } + - match: { aggregations.m_terms.buckets.1.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.1" } - match: { aggregations.m_terms.buckets.1.doc_count: 3 } - - match: { aggregations.m_terms.buckets.2.key_as_string: "{k8s.pod.uid=947e4ced-1786-4e53-9e0c-5c447e959507, metricset=pod}|10.10.55.2" } + - match: { aggregations.m_terms.buckets.2.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.2" } - match: { aggregations.m_terms.buckets.2.doc_count: 1 } +--- +"multi_terms aggregation with time_series aggregation": + - skip: + version: " - 8.12.99" + reason: "multi_terms for time series aggregation fixed in 8.13.0" + + - do: + search: + index: test + body: + aggs: + ts: + time_series: {} + m_terms: + multi_terms: + collect_mode: breadth_first + terms: + - field: k8s.pod.name + - field: k8s.pod.ip + aggs: + max_value: + max: + field: val + - length: { aggregations.ts.buckets: 2 } + - length: { aggregations.m_terms.buckets: 3 } + --- "Auto date histogram on counter": - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml index abb0d038cb2c3..684d554f08e58 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml @@ -15,6 +15,18 @@ setup: wait_for_resources_created: true timeout: "1m" + - do: + indices.create: + index: test-events + body: + mappings: + properties: + "@timestamp": + type: date + format: epoch_second + events: + type: counted_keyword + - do: bulk: refresh: wait_for @@ -105,6 +117,15 @@ setup: - {"@timestamp": "1698624000", "Executable": {"build": {"id": "c5f89ea1c68710d2a493bb604c343a92c4f8ddeb"}, "file": {"name": "vmlinux"}}, "Symbolization": {"next_time": "4852491791"}, "ecs": {"version": "1.12.0"}} - {"create": {"_index": "profiling-hosts", "_id": "eLH27YsBj2lLi3tJYlvr"}} - {"profiling.project.id": 100, "host.id": "8457605156473051743", "@timestamp": 1700504426, "ecs.version": "1.12.0", "profiling.agent.build_timestamp": 1688111067, "profiling.instance.private_ipv4s": ["192.168.1.2"], "ec2.instance_life_cycle": "on-demand", "profiling.agent.config.map_scale_factor": 0, "ec2.instance_type": "i3.2xlarge", "profiling.host.ip": "192.168.1.2", "profiling.agent.config.bpf_log_level": 0, "profiling.host.sysctl.net.core.bpf_jit_enable": 1, "profiling.agent.config.file": "/etc/prodfiler/prodfiler.conf", "ec2.local_ipv4": "192.168.1.2", "profiling.agent.config.no_kernel_version_check": false, "profiling.host.machine": "x86_64", "profiling.host.tags": ["cloud_provider:aws", "cloud_environment:qa", "cloud_region:eu-west-1"], "profiling.agent.config.probabilistic_threshold": 100, "profiling.agent.config.disable_tls": false, "profiling.agent.config.tracers": "all", "profiling.agent.start_time": 1700090045589, "profiling.agent.config.max_elements_per_interval": 800, "ec2.placement.region": "eu-west-1", "profiling.agent.config.present_cpu_cores": 8, "profiling.host.kernel_version": "9.9.9-0-aws", "profiling.agent.config.bpf_log_size": 65536, "profiling.agent.config.known_traces_entries": 65536, "profiling.host.sysctl.kernel.unprivileged_bpf_disabled": 1, "profiling.agent.config.verbose": false, "profiling.agent.config.probabilistic_interval": "1m0s", "ec2.placement.availability_zone_id": "euw1-az1", "ec2.security_groups": "", "ec2.local_hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "ec2.placement.availability_zone": "eu-west-1c", "profiling.agent.config.upload_symbols": false, "profiling.host.sysctl.kernel.bpf_stats_enabled": 0, "profiling.host.name": "ip-192-168-1-2", "ec2.mac": "00:11:22:33:44:55", "profiling.host.kernel_proc_version": "Linux version 9.9.9-0-aws", "profiling.agent.config.cache_directory": "/var/cache/optimyze/", "profiling.agent.version": "v8.12.0", "ec2.hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "profiling.agent.config.elastic_mode": false, "ec2.ami_id": "ami-aaaaaaaaaaa", "ec2.instance_id": "i-0b999999999999999" } + - {"index": {"_index": "test-events"}} + - {"@timestamp": "1700504427", "events": ["S07KmaoGhvNte78xwwRbZQ"]} +--- +teardown: + - do: + cluster.put_settings: + body: + persistent: + xpack.profiling.templates.enabled: false --- "Test Status": @@ -144,7 +165,7 @@ setup: - match: { stack_traces.S07KmaoGhvNte78xwwRbZQ.count: 1} --- -"Test flamegraph": +"Test flamegraph from profiling-events": - do: profiling.flamegraph: body: > @@ -170,9 +191,29 @@ setup: - match: { Size: 47} --- -teardown: +"Test flamegraph from test-events": - do: - cluster.put_settings: - body: - persistent: - xpack.profiling.templates.enabled: false + profiling.flamegraph: + body: > + { + "sample_size": 20000, + "indices": "test-events", + "stacktrace_ids_field": "events", + "requested_duration": 86400, + "query": { + "bool": { + "filter": [ + { + "range": { + "@timestamp": { + "gte": "2023-11-20", + "lt": "2023-11-21", + "format": "yyyy-MM-dd" + } + } + } + ] + } + } + } + - match: { Size: 47} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml index 8044e9bc3b8ab..66ce482f70603 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml @@ -104,13 +104,13 @@ document level security on tag: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 1} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- document level security on dimension: - skip: - version: " - 8.0.99" + version: " - 8.11.99" reason: _tsid support introduced in 8.1.0 features: headers @@ -151,7 +151,7 @@ document level security on dimension: - match: { hits.total.value: 4 } - length: { aggregations.tsids.buckets: 1 } - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/users/40_query.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/users/40_query.yml new file mode 100644 index 0000000000000..d6258a96650e9 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/users/40_query.yml @@ -0,0 +1,125 @@ +--- +setup: + - skip: + features: headers + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "manage_users_role" + body: > + { + "cluster": ["manage_security"] + } + + - do: + security.put_user: + username: "users_admin" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "manage_users_role" ], + "full_name" : "Read User" + } + +--- +teardown: + - do: + security.delete_role: + name: "manage_users_role" + ignore: 404 + + - do: + security.delete_user: + username: "users_admin" + ignore: 404 + + - do: + security.delete_user: + username: "test_user_1" + ignore: 404 + + - do: + security.delete_user: + username: "test_user_2" + ignore: 404 + +--- +"Test query user": + + - do: + headers: + Authorization: "Basic dXNlcnNfYWRtaW46eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # users_admin + security.put_user: + username: "test_user_1" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "test-role-1" ], + "full_name" : "Test User 1" + } + - match: { "created": true } + + - do: + headers: + Authorization: "Basic dXNlcnNfYWRtaW46eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # users_admin + security.put_user: + username: "test_user_2" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "test-role-2" ], + "full_name" : "Test User 2" + } + - match: { "created": true } + + # empty body works just like match_all + - do: + headers: + Authorization: "Basic dXNlcnNfYWRtaW46eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # users_admin + security.query_user: + body: { } + - match: { total: 3 } + - match: { count: 3 } + + # match_all + - do: + headers: + Authorization: "Basic dXNlcnNfYWRtaW46eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # users_admin + security.query_user: + body: > + { + "query": { "match_all": {} } + } + - match: { total: 3 } + - match: { count: 3 } + + # Wildcard + - do: + headers: + Authorization: "Basic dXNlcnNfYWRtaW46eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # users_admin + security.query_user: + body: > + { + "query": { "wildcard": {"roles": "test-role-*"} } + } + - match: { total: 2 } + - match: { count: 2 } + + - do: + headers: + Authorization: "Basic dXNlcnNfYWRtaW46eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # users_admin + security.query_user: + body: > + { + "query": { "wildcard": {"roles": "test-role-*"} }, + "sort": [ {"username": {"order": "desc"}} ], + "from": 1, + "size": 1 + } + - match: { total: 2 } + - match: { count: 1 } + - match: { users.0.username: "test_user_1" } diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/TextStructurePlugin.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/TextStructurePlugin.java index b7d9117a3f9dc..2a2fe1ea5a55a 100644 --- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/TextStructurePlugin.java +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/TextStructurePlugin.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; @@ -29,6 +30,7 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; /** @@ -48,7 +50,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList(new RestFindStructureAction(), new RestTestGrokPatternAction()); } diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java index 1fb1b3ac0bc5c..74ee8ea88e04f 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformChainIT.java @@ -189,7 +189,7 @@ private void testChainedTransforms(final int numTransforms) throws Exception { assertBusy(() -> { // Verify that each transform processed an expected number of documents. for (String transformId : transformIds) { - Map stats = getTransformStats(transformId); + Map stats = getBasicTransformStats(transformId); assertThat( "Stats were: " + stats, XContentMapValues.extractValue(stats, "stats", "documents_processed"), diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java index 1302f20838c4a..394732742e528 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java @@ -82,76 +82,76 @@ public void cleanTransforms() throws Exception { } public void testTransformCrud() throws Exception { - String indexName = "basic-crud-reviews"; - String transformId = "transform-crud"; - createReviewsIndex(indexName, 100, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); - - Map groups = new HashMap<>(); - groups.put("by-day", createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null)); - groups.put("by-user", new TermsGroupSource("user_id", null, false)); - groups.put("by-business", new TermsGroupSource("business_id", null, false)); - - AggregatorFactories.Builder aggs = AggregatorFactories.builder() - .addAggregator(AggregationBuilders.avg("review_score").field("stars")) - .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); - - TransformConfig config = createTransformConfigBuilder( - transformId, - "reviews-by-user-business-day", - QueryConfig.matchAll(), - indexName - ).setPivotConfig(createPivotConfig(groups, aggs)).build(); - - putTransform(transformId, Strings.toString(config), RequestOptions.DEFAULT); - startTransform(config.getId(), RequestOptions.DEFAULT); - - waitUntilCheckpoint(config.getId(), 1L); + var transformId = "transform-crud"; + createStoppedTransform("basic-crud-reviews", transformId); + assertBusy(() -> { assertEquals("stopped", getTransformState(transformId)); }); - stopTransform(config.getId()); - assertBusy(() -> { assertEquals("stopped", getTransformStats(config.getId()).get("state")); }); - - var storedConfig = getTransform(config.getId()); + var storedConfig = getTransform(transformId); assertThat(storedConfig.get("version"), equalTo(TransformConfigVersion.CURRENT.toString())); Instant now = Instant.now(); long createTime = (long) storedConfig.get("create_time"); assertTrue("[create_time] is not before current time", Instant.ofEpochMilli(createTime).isBefore(now)); - deleteTransform(config.getId()); + deleteTransform(transformId); } - public void testContinuousTransformCrud() throws Exception { - String indexName = "continuous-crud-reviews"; - String transformId = "transform-continuous-crud"; + private void createStoppedTransform(String indexName, String transformId) throws Exception { createReviewsIndex(indexName, 100, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); - Map groups = new HashMap<>(); - groups.put("by-day", createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null)); - groups.put("by-user", new TermsGroupSource("user_id", null, false)); - groups.put("by-business", new TermsGroupSource("business_id", null, false)); + var groups = Map.of( + "by-day", + createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null), + "by-user", + new TermsGroupSource("user_id", null, false), + "by-business", + new TermsGroupSource("business_id", null, false) + ); - AggregatorFactories.Builder aggs = AggregatorFactories.builder() + var aggs = AggregatorFactories.builder() .addAggregator(AggregationBuilders.avg("review_score").field("stars")) .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); - TransformConfig config = createTransformConfigBuilder( - transformId, - "reviews-by-user-business-day", - QueryConfig.matchAll(), - indexName - ).setPivotConfig(createPivotConfig(groups, aggs)) - .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) - .setSettings(new SettingsConfig.Builder().setAlignCheckpoints(false).build()) + var config = createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryConfig.matchAll(), indexName) + .setPivotConfig(createPivotConfig(groups, aggs)) .build(); putTransform(transformId, Strings.toString(config), RequestOptions.DEFAULT); startTransform(config.getId(), RequestOptions.DEFAULT); waitUntilCheckpoint(config.getId(), 1L); - var transformStats = getTransformStats(config.getId()); + stopTransform(config.getId()); + } + + /** + * Verify the basic stats API, which includes state, health, and optionally progress (if it exists). + * These are required for Kibana 8.13+. + */ + @SuppressWarnings("unchecked") + public void testBasicTransformStats() throws Exception { + var transformId = "transform-basic-stats"; + createStoppedTransform("basic-stats-reviews", transformId); + var transformStats = getBasicTransformStats(transformId); + + assertBusy(() -> assertEquals("stopped", XContentMapValues.extractValue("state", transformStats))); + assertEquals("green", XContentMapValues.extractValue("health.status", transformStats)); + assertThat( + "percent_complete is not 100.0", + XContentMapValues.extractValue("checkpointing.next.checkpoint_progress.percent_complete", transformStats), + equalTo(100.0) + ); + + deleteTransform(transformId); + } + + public void testContinuousTransformCrud() throws Exception { + var transformId = "transform-continuous-crud"; + var indexName = "continuous-crud-reviews"; + createContinuousTransform(indexName, transformId); + var transformStats = getBasicTransformStats(transformId); assertThat(transformStats.get("state"), equalTo("started")); int docsIndexed = (Integer) XContentMapValues.extractValue("stats.documents_indexed", transformStats); - var storedConfig = getTransform(config.getId()); + var storedConfig = getTransform(transformId); assertThat(storedConfig.get("version"), equalTo(TransformConfigVersion.CURRENT.toString())); Instant now = Instant.now(); long createTime = (long) storedConfig.get("create_time"); @@ -161,17 +161,66 @@ public void testContinuousTransformCrud() throws Exception { long timeStamp = Instant.now().toEpochMilli() - 1_000; long user = 42; indexMoreDocs(timeStamp, user, indexName); - waitUntilCheckpoint(config.getId(), 2L); + waitUntilCheckpoint(transformId, 2L); // Assert that we wrote the new docs assertThat( - (Integer) XContentMapValues.extractValue("stats.documents_indexed", getTransformStats(config.getId())), + (Integer) XContentMapValues.extractValue("stats.documents_indexed", getBasicTransformStats(transformId)), greaterThan(docsIndexed) ); - stopTransform(config.getId()); - deleteTransform(config.getId()); + stopTransform(transformId); + deleteTransform(transformId); + } + + private void createContinuousTransform(String indexName, String transformId) throws Exception { + createReviewsIndex(indexName, 100, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); + + var groups = Map.of( + "by-day", + createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null), + "by-user", + new TermsGroupSource("user_id", null, false), + "by-business", + new TermsGroupSource("business_id", null, false) + ); + + var aggs = AggregatorFactories.builder() + .addAggregator(AggregationBuilders.avg("review_score").field("stars")) + .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); + + var config = createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryConfig.matchAll(), indexName) + .setPivotConfig(createPivotConfig(groups, aggs)) + .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) + .setSettings(new SettingsConfig.Builder().setAlignCheckpoints(false).build()) + .build(); + + putTransform(transformId, Strings.toString(config), RequestOptions.DEFAULT); + startTransform(config.getId(), RequestOptions.DEFAULT); + + waitUntilCheckpoint(config.getId(), 1L); + } + + /** + * Verify the basic stats API, which includes state, health, and optionally progress (if it exists). + * These are required for Kibana 8.13+. + */ + @SuppressWarnings("unchecked") + public void testBasicContinuousTransformStats() throws Exception { + var transformId = "transform-continuous-basic-stats"; + createContinuousTransform("continuous-basic-stats-reviews", transformId); + var transformStats = getBasicTransformStats(transformId); + + assertEquals("started", XContentMapValues.extractValue("state", transformStats)); + assertEquals("green", XContentMapValues.extractValue("health.status", transformStats)); + + // We aren't testing for 'checkpointing.next.checkpoint_progress.percent_complete'. + // It's difficult to get the integration test to reliably call the stats API while that data is available, since continuous + // transforms start and finish the next checkpoint quickly (<1ms). + + stopTransform(transformId); + deleteTransform(transformId); } public void testContinuousTransformUpdate() throws Exception { @@ -197,7 +246,7 @@ public void testContinuousTransformUpdate() throws Exception { waitUntilCheckpoint(config.getId(), 1L); assertThat(getTransformState(config.getId()), oneOf("started", "indexing")); - int docsIndexed = (Integer) XContentMapValues.extractValue("stats.documents_indexed", getTransformStats(config.getId())); + int docsIndexed = (Integer) XContentMapValues.extractValue("stats.documents_indexed", getBasicTransformStats(config.getId())); var storedConfig = getTransform(config.getId()); assertThat(storedConfig.get("version"), equalTo(TransformConfigVersion.CURRENT.toString())); @@ -238,7 +287,7 @@ public void testContinuousTransformUpdate() throws Exception { // Since updates are loaded on checkpoint start, we should see the updated config on this next run waitUntilCheckpoint(config.getId(), 2L); - int numDocsAfterCp2 = (Integer) XContentMapValues.extractValue("stats.documents_indexed", getTransformStats(config.getId())); + int numDocsAfterCp2 = (Integer) XContentMapValues.extractValue("stats.documents_indexed", getBasicTransformStats(config.getId())); assertThat(numDocsAfterCp2, greaterThan(docsIndexed)); Request searchRequest = new Request("GET", dest + "/_search"); @@ -326,7 +375,7 @@ public void testStopWaitForCheckpoint() throws Exception { // wait until transform has been triggered and indexed at least 1 document assertBusy(() -> { - var stateAndStats = getTransformStats(config.getId()); + var stateAndStats = getBasicTransformStats(config.getId()); assertThat((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats), greaterThan(1)); }); @@ -338,7 +387,7 @@ public void testStopWaitForCheckpoint() throws Exception { // Even though we are continuous, we should be stopped now as we needed to stop at the first checkpoint assertBusy(() -> { - var stateAndStats = getTransformStats(config.getId()); + var stateAndStats = getBasicTransformStats(config.getId()); assertThat(stateAndStats.get("state"), equalTo("stopped")); assertThat((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats), equalTo(1000)); }); @@ -356,12 +405,12 @@ public void testStopWaitForCheckpoint() throws Exception { stopTransform(transformId, waitForCompletion, null, true); assertBusy(() -> { - var stateAndStats = getTransformStats(config.getId()); + var stateAndStats = getBasicTransformStats(config.getId()); assertThat(stateAndStats.get("state"), equalTo("stopped")); }); } - var stateAndStats = getTransformStats(config.getId()); + var stateAndStats = getBasicTransformStats(config.getId()); assertThat(stateAndStats.get("state"), equalTo("stopped")); // Despite indexing new documents into the source index, the number of documents in the destination index stays the same. assertThat((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats), equalTo(1000)); @@ -399,10 +448,7 @@ public void testContinuousTransformRethrottle() throws Exception { putTransform(transformId, Strings.toString(config), RequestOptions.DEFAULT); startTransform(config.getId(), RequestOptions.DEFAULT); - assertBusy(() -> { - var stateAndStats = getTransformStats(config.getId()); - assertThat(stateAndStats.get("state"), equalTo("indexing")); - }); + assertBusy(() -> { assertThat(getTransformState(config.getId()), equalTo("indexing")); }); // test randomly: with explicit settings and reset to default String reqsPerSec = randomBoolean() ? "1000" : "null"; @@ -421,7 +467,7 @@ public void testContinuousTransformRethrottle() throws Exception { waitUntilCheckpoint(config.getId(), 1L); assertThat(getTransformState(config.getId()), equalTo("started")); - var transformStats = getTransformStats(config.getId()); + var transformStats = getBasicTransformStats(config.getId()); int docsIndexed = (Integer) XContentMapValues.extractValue("stats.documents_indexed", transformStats); int pagesProcessed = (Integer) XContentMapValues.extractValue("stats.pages_processed", transformStats); diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java index 319090a50b21a..e46d32295078f 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformInsufficientPermissionsIT.java @@ -571,14 +571,14 @@ private TransformConfig createConfig( } private void assertGreen(String transformId) throws IOException { - Map stats = getTransformStats(transformId); + Map stats = getBasicTransformStats(transformId); assertThat("Stats were: " + stats, extractValue(stats, "health", "status"), is(equalTo(GREEN))); assertThat("Stats were: " + stats, extractValue(stats, "health", "issues"), is(nullValue())); } @SuppressWarnings("unchecked") private void assertRed(String transformId, String... expectedHealthIssueDetails) throws IOException { - Map stats = getTransformStats(transformId); + Map stats = getBasicTransformStats(transformId); assertThat("Stats were: " + stats, extractValue(stats, "health", "status"), is(equalTo(RED))); List issues = (List) extractValue(stats, "health", "issues"); assertThat("Stats were: " + stats, issues, hasSize(expectedHealthIssueDetails.length)); diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index 9c4241fa88ef5..eed849d35ea44 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -62,6 +62,7 @@ import java.util.function.Function; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.transform.TransformField.BASIC_STATS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.core.Is.is; @@ -256,8 +257,19 @@ protected Map getTransformStats(String id) throws IOException { return stats.get(0); } + @SuppressWarnings("unchecked") + protected Map getBasicTransformStats(String id) throws IOException { + var request = new Request("GET", TRANSFORM_ENDPOINT + id + "/_stats"); + request.addParameter(BASIC_STATS.getPreferredName(), "true"); + request.setOptions(RequestOptions.DEFAULT); + Response response = client().performRequest(request); + List> stats = (List>) XContentMapValues.extractValue("transforms", entityAsMap(response)); + assertThat(stats, hasSize(1)); + return stats.get(0); + } + protected String getTransformState(String id) throws IOException { - return (String) getTransformStats(id).get("state"); + return (String) getBasicTransformStats(id).get("state"); } @SuppressWarnings("unchecked") @@ -285,7 +297,7 @@ protected void waitUntilCheckpoint(String id, long checkpoint, TimeValue waitTim assertBusy( () -> assertEquals( checkpoint, - ((Integer) XContentMapValues.extractValue("checkpointing.last.checkpoint", getTransformStats(id))).longValue() + ((Integer) XContentMapValues.extractValue("checkpointing.last.checkpoint", getBasicTransformStats(id))).longValue() ), waitTime.getMillis(), TimeUnit.MILLISECONDS diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java index e27d6a224802c..9f4a15029f05f 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsingSearchRuntimeFieldsIT.java @@ -146,10 +146,7 @@ public void testPivotTransform() throws Exception { waitUntilCheckpoint(config.getId(), 1L); stopTransform(config.getId()); - assertBusy(() -> { - var stats = getTransformStats(config.getId()); - assertEquals("stopped", stats.get("state")); - }); + assertBusy(() -> { assertEquals("stopped", getTransformState(config.getId())); }); refreshIndex(destIndexName, RequestOptions.DEFAULT); // Verify destination index mappings diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java index 37c3d774e59e6..d73dc2a4e3aac 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java @@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit; import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.core.transform.TransformField.BASIC_STATS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasEntry; @@ -158,6 +159,84 @@ public void testGetAndGetStats() throws Exception { stopTransform("pivot_continuous", false); } + /** + * Verify the basic stats API, which includes state, health, and optionally progress (if it exists). + * These are required for Kibana 8.13+. + */ + @SuppressWarnings("unchecked") + public void testGetAndGetBasicStats() throws Exception { + createPivotReviewsTransform("pivot_1", "pivot_reviews_1", null); + createContinuousPivotReviewsTransform("pivot_continuous", "pivot_reviews_continuous", null); + + startAndWaitForTransform("pivot_1", "pivot_reviews_1"); + startAndWaitForContinuousTransform("pivot_continuous", "pivot_reviews_continuous", null); + + stopTransform("pivot_1", false); + // Alternate testing between admin and lowly user, as both should be able to get the configs and stats + var authHeader = randomFrom(BASIC_AUTH_VALUE_TRANSFORM_USER, BASIC_AUTH_VALUE_TRANSFORM_ADMIN); + + // Check all the different ways to retrieve transform stats + var basicStats = "_stats?" + BASIC_STATS.getPreferredName() + "=true"; + var getRequest = createRequestWithAuthAndTimeout("GET", getTransformEndpoint() + basicStats, authHeader, randomTimeout()); + var stats = entityAsMap(client().performRequest(getRequest)); + assertEquals(2, XContentMapValues.extractValue("count", stats)); + getRequest = createRequestWithAuthAndTimeout("GET", getTransformEndpoint() + "_all/" + basicStats, authHeader, randomTimeout()); + stats = entityAsMap(client().performRequest(getRequest)); + assertEquals(2, XContentMapValues.extractValue("count", stats)); + getRequest = createRequestWithAuthAndTimeout("GET", getTransformEndpoint() + "*/" + basicStats, authHeader, randomTimeout()); + stats = entityAsMap(client().performRequest(getRequest)); + assertEquals(2, XContentMapValues.extractValue("count", stats)); + getRequest = createRequestWithAuthAndTimeout("GET", getTransformEndpoint() + "pivot_1/" + basicStats, authHeader, randomTimeout()); + stats = entityAsMap(client().performRequest(getRequest)); + assertEquals(1, XContentMapValues.extractValue("count", stats)); + getRequest = createRequestWithAuthAndTimeout("GET", getTransformEndpoint() + "pivot_*/" + basicStats, authHeader, randomTimeout()); + stats = entityAsMap(client().performRequest(getRequest)); + assertEquals(2, XContentMapValues.extractValue("count", stats)); + + // verify pivot_1 has basic stats + getRequest = createRequestWithAuthAndTimeout("GET", getTransformEndpoint() + "pivot_1/" + basicStats, authHeader, randomTimeout()); + stats = entityAsMap(client().performRequest(getRequest)); + assertEquals(1, XContentMapValues.extractValue("count", stats)); + var transform = ((List>) XContentMapValues.extractValue("transforms", stats)).get(0); + + // verify state exists + assertEquals("stopped", XContentMapValues.extractValue("state", transform)); + + // verify health exists + assertEquals("green", XContentMapValues.extractValue("health.status", transform)); + + // verify checkpointing exists + assertThat( + "percent_complete is not 100.0", + XContentMapValues.extractValue("checkpointing.next.checkpoint_progress.percent_complete", transform), + equalTo(100.0) + ); + + // verify pivot_continuous has basic stats + getRequest = createRequestWithAuthAndTimeout( + "GET", + getTransformEndpoint() + "pivot_continuous/" + basicStats, + authHeader, + randomTimeout() + ); + stats = entityAsMap(client().performRequest(getRequest)); + assertEquals(1, XContentMapValues.extractValue("count", stats)); + transform = ((List>) XContentMapValues.extractValue("transforms", stats)).get(0); + + // verify state exists + assertEquals("started", XContentMapValues.extractValue("state", transform)); + + // verify health exists + assertEquals("green", XContentMapValues.extractValue("health.status", transform)); + + // We aren't testing for 'checkpointing.next.checkpoint_progress.percent_complete'. + // It's difficult to get the integration test to reliably call the stats API while that data is available, since continuous + // transforms start and finish the next checkpoint quickly (<1ms). + + stopTransform("pivot_continuous", false); + } + + @SuppressWarnings("unchecked") public void testGetAndGetStatsForTransformWithoutConfig() throws Exception { createPivotReviewsTransform("pivot_1", "pivot_reviews_1", null); createPivotReviewsTransform("pivot_2", "pivot_reviews_2", null); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index 6aeca79b4aa17..50b078730063d 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -28,6 +28,7 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInRelativeOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -920,7 +921,7 @@ public void testPivotWithTermsAgg() throws Exception { assertEquals(3, XContentMapValues.extractValue("_all.total.docs.count", indexStats)); // get and check some term results - Map searchResult = getAsMap(transformIndex + "/_search?q=every_2:2.0"); + Map searchResult = getAsOrderedMap(transformIndex + "/_search?q=every_2:2.0"); assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult)); Map commonUsers = (Map) ((List) XContentMapValues.extractValue( @@ -944,9 +945,8 @@ public void testPivotWithTermsAgg() throws Exception { searchResult )).get(0); assertThat(commonUsersDesc, is(not(nullValue()))); - // 3 user names latest in lexicographic order (user_7, user_8, user_9) are selected properly but their order is not preserved. - // See https://github.com/elastic/elasticsearch/issues/104847 for more information. - assertThat(commonUsersDesc, equalTo(Map.of("user_7", 6, "user_9", 2, "user_8", 8))); + // 3 user names latest in lexicographic order (user_9, user_8, user_7) are selected properly and their order is preserved. + assertThat(commonUsersDesc.keySet(), containsInRelativeOrder("user_9", "user_8", "user_7")); Map rareUsers = (Map) ((List) XContentMapValues.extractValue( "hits.hits._source.rare_users", searchResult diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java index 77a0cc7f79841..5872e78e133d1 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java @@ -351,7 +351,7 @@ private void testTransformLifecycle(QueryBuilder query, long expectedHitCount) t assertTrue(response.isAcknowledged()); } assertBusy(() -> { - GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(transformId, TIMEOUT); + GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(transformId, TIMEOUT, true); GetTransformStatsAction.Response response = client().execute(GetTransformStatsAction.INSTANCE, request).actionGet(); assertThat("Stats were: " + response.getTransformsStats(), response.getTransformsStats(), hasSize(1)); assertThat(response.getTransformsStats().get(0).getState(), is(equalTo(TransformStats.State.STOPPED))); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java index 0130b33837877..1ce2299df33fb 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java @@ -41,7 +41,11 @@ protected Settings nodeSettings() { } public void testGetTransformStats() { - GetTransformStatsAction.Request request = new GetTransformStatsAction.Request("_all", AcknowledgedRequest.DEFAULT_ACK_TIMEOUT); + GetTransformStatsAction.Request request = new GetTransformStatsAction.Request( + "_all", + AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + true + ); GetTransformStatsAction.Response response = client().execute(GetTransformStatsAction.INSTANCE, request).actionGet(); assertThat(response.getTransformsStats(), is(empty())); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index cb118cead1dc9..bde5576fb0c92 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.indices.AssociatedIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; @@ -111,6 +112,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -173,7 +175,8 @@ public List getRestHandlers( final IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter, final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster + final Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return Arrays.asList( diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java index f7e60b13b50a6..ce15098775d9a 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java @@ -134,6 +134,15 @@ protected void taskOperation( listener.onResponse(new Response(Collections.emptyList())); return; } + + if (request.isBasic()) { + ActionListener.completeWith( + listener, + () -> new Response(Collections.singletonList(deriveStats(transformTask, transformTask.deriveBasicCheckpointingInfo()))) + ); + return; + } + transformTask.getCheckpointingInfo( transformCheckpointService, new ParentTaskAssigningClient(client, parentTaskId), @@ -301,7 +310,7 @@ private void collectStatsForTransformsWithoutTasks( List allStateAndStats = new ArrayList<>(response.getTransformsStats()); addCheckpointingInfoForTransformsWithoutTasks( parentTaskId, - request.getTimeout(), + request, allStateAndStats, statsForTransformsWithoutTasks, transformsWaitingForAssignment, @@ -337,26 +346,33 @@ private void collectStatsForTransformsWithoutTasks( private void populateSingleStoppedTransformStat( TransformStoredDoc transform, TaskId parentTaskId, - TimeValue timeout, + Request request, ActionListener listener ) { - transformCheckpointService.getCheckpointingInfo( - new ParentTaskAssigningClient(client, parentTaskId), - timeout, - transform.getId(), - transform.getTransformState().getCheckpoint(), - transform.getTransformState().getPosition(), - transform.getTransformState().getProgress(), - ActionListener.wrap(infoBuilder -> listener.onResponse(infoBuilder.build()), e -> { - logger.warn("Failed to retrieve checkpointing info for transform [" + transform.getId() + "]", e); - listener.onResponse(TransformCheckpointingInfo.EMPTY); - }) - ); + if (request.isBasic()) { + ActionListener.completeWith( + listener, + () -> TransformCheckpointService.deriveBasicCheckpointingInfo(transform.getTransformState()) + ); + } else { + transformCheckpointService.getCheckpointingInfo( + new ParentTaskAssigningClient(client, parentTaskId), + request.getTimeout(), + transform.getId(), + transform.getTransformState().getCheckpoint(), + transform.getTransformState().getPosition(), + transform.getTransformState().getProgress(), + ActionListener.wrap(infoBuilder -> listener.onResponse(infoBuilder.build()), e -> { + logger.warn("Failed to retrieve checkpointing info for transform [" + transform.getId() + "]", e); + listener.onResponse(TransformCheckpointingInfo.EMPTY); + }) + ); + } } private void addCheckpointingInfoForTransformsWithoutTasks( TaskId parentTaskId, - TimeValue timeout, + Request request, List allStateAndStats, List statsForTransformsWithoutTasks, Set transformsWaitingForAssignment, @@ -373,7 +389,7 @@ private void addCheckpointingInfoForTransformsWithoutTasks( AtomicBoolean isExceptionReported = new AtomicBoolean(false); statsForTransformsWithoutTasks.forEach( - stat -> populateSingleStoppedTransformStat(stat, parentTaskId, timeout, ActionListener.wrap(checkpointingInfo -> { + stat -> populateSingleStoppedTransformStat(stat, parentTaskId, request, ActionListener.wrap(checkpointingInfo -> { synchronized (allStateAndStats) { if (transformsWaitingForAssignment.contains(stat.getId())) { Assignment assignment = TransformNodes.getAssignment(stat.getId(), clusterState); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointService.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointService.java index 0006a79b6a2b8..0826aaff9deab 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointService.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointService.java @@ -15,10 +15,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointStats; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo.TransformCheckpointingInfoBuilder; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; +import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; @@ -109,4 +112,25 @@ public void getCheckpointingInfo( listener.onFailure(new CheckpointException("Failed to retrieve configuration", transformError)); })); } + + /** + * Derives basic checkpointing stats for a stopped transform. This does not make a call to obtain any additional information. + * This will only read checkpointing information from the TransformState. + * + * @param transformState the current state of the Transform + * @return basic checkpointing info, including id, position, and progress of the Next Checkpoint and the id of the Last Checkpoint. + */ + public static TransformCheckpointingInfo deriveBasicCheckpointingInfo(TransformState transformState) { + return new TransformCheckpointingInfo(lastCheckpointStats(transformState), nextCheckpointStats(transformState), 0L, null, null); + } + + private static TransformCheckpointStats lastCheckpointStats(TransformState transformState) { + return new TransformCheckpointStats(transformState.getCheckpoint(), null, null, 0L, 0L); + } + + private static TransformCheckpointStats nextCheckpointStats(TransformState transformState) { + // getCheckpoint is the last checkpoint. if we're at zero then we'd only call to get the zeroth checkpoint (see getCheckpointInfo) + var checkpoint = transformState.getCheckpoint() != 0 ? transformState.getCheckpoint() + 1 : 0; + return new TransformCheckpointStats(checkpoint, transformState.getPosition(), transformState.getProgress(), 0L, 0L); + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java index b68afa8db75fb..409afb125c323 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java @@ -62,7 +62,7 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c GetTransformAction.Request request = new GetTransformAction.Request(id); request.setAllowNoResources(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true)); - GetTransformStatsAction.Request statsRequest = new GetTransformStatsAction.Request(id, null); + GetTransformStatsAction.Request statsRequest = new GetTransformStatsAction.Request(id, null, false); statsRequest.setAllowNoMatch(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true)); if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java index dfc1580cd48e5..72d72163fd4df 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java @@ -25,6 +25,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.transform.TransformField.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.transform.TransformField.BASIC_STATS; @ServerlessScope(Scope.PUBLIC) public class RestGetTransformStatsAction extends BaseRestHandler { @@ -41,7 +42,8 @@ public List routes() { protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) { String id = restRequest.param(TransformField.ID.getPreferredName()); TimeValue timeout = restRequest.paramAsTime(TransformField.TIMEOUT.getPreferredName(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT); - GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(id, timeout); + var basic = restRequest.paramAsBoolean(BASIC_STATS.getPreferredName(), false); + GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(id, timeout, basic); request.setAllowNoMatch(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true)); if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) { request.setPageParams( diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java index 6ab7e7764b187..30b50a1460f70 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java @@ -211,6 +211,24 @@ public void getCheckpointingInfo( ); } + /** + * Derives basic checkpointing stats. This does not make a call to obtain any additional information. + * This will only read checkpointing information from this TransformTask. + * + * @return basic checkpointing info, including id, position, and progress of the Next Checkpoint and the id of the Last Checkpoint. + */ + public TransformCheckpointingInfo deriveBasicCheckpointingInfo() { + var transformIndexer = getIndexer(); + if (transformIndexer == null) { + return TransformCheckpointingInfo.EMPTY; + } + return new TransformCheckpointingInfo.TransformCheckpointingInfoBuilder().setLastCheckpoint(transformIndexer.getLastCheckpoint()) + .setNextCheckpoint(transformIndexer.getNextCheckpoint()) + .setNextCheckpointPosition(transformIndexer.getPosition()) + .setNextCheckpointProgress(transformIndexer.getProgress()) + .build(); + } + /** * Starts the transform and schedules it to be triggered in the future. * diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java index 1c6c411020d49..a851e4a47f1cc 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.search.aggregations.Aggregation; @@ -38,10 +39,10 @@ import org.elasticsearch.xpack.transform.transforms.IDGenerator; import org.elasticsearch.xpack.transform.utils.OutputFieldNameConverter; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -106,7 +107,7 @@ public static Stream> extractCompositeAggregationResults( progress.incrementDocsProcessed(bucket.getDocCount()); progress.incrementDocsIndexed(1L); - Map document = new HashMap<>(); + Map document = new LinkedHashMap<>(); // generator to create unique but deterministic document ids, so we // - do not create duplicates if we re-run after failure // - update documents @@ -223,7 +224,7 @@ static void updateDocument(Map document, String fieldName, Objec throw new AggregationExtractionException("mixed object types of nested and non-nested fields [{}]", fieldName); } } else { - Map newMap = new HashMap<>(); + Map newMap = new LinkedHashMap<>(); internalMap.put(token, newMap); internalMap = newMap; } @@ -284,7 +285,7 @@ static class MultiValueAggExtractor implements AggValueExtractor { @Override public Object value(Aggregation agg, Map fieldTypeMap, String lookupFieldPrefix) { MultiValueAggregation aggregation = (MultiValueAggregation) agg; - Map extracted = new HashMap<>(); + Map extracted = new LinkedHashMap<>(); for (String valueName : aggregation.valueNames()) { List valueAsStrings = aggregation.getValuesAsStrings(valueName); @@ -302,7 +303,7 @@ static class NumericMultiValueAggExtractor implements AggValueExtractor { @Override public Object value(Aggregation agg, Map fieldTypeMap, String lookupFieldPrefix) { MultiValue aggregation = (MultiValue) agg; - Map extracted = new HashMap<>(); + Map extracted = new LinkedHashMap<>(); String fieldLookupPrefix = (lookupFieldPrefix.isEmpty() ? agg.getName() : lookupFieldPrefix + "." + agg.getName()) + "."; for (String valueName : aggregation.valueNames()) { @@ -322,7 +323,7 @@ static class PercentilesAggExtractor implements AggValueExtractor { @Override public Object value(Aggregation agg, Map fieldTypeMap, String lookupFieldPrefix) { Percentiles aggregation = (Percentiles) agg; - HashMap percentiles = new HashMap<>(); + Map percentiles = new LinkedHashMap<>(); for (Percentile p : aggregation) { // in case of sparse data percentiles might not have data, in this case it returns NaN, @@ -360,16 +361,10 @@ public Object value(Aggregation agg, Map fieldTypeMap, String lo return aggregation.getDocCount(); } - HashMap nested = new HashMap<>(); + var subAggLookupFieldPrefix = lookupFieldPrefix.isEmpty() ? agg.getName() : lookupFieldPrefix + "." + agg.getName(); + Map nested = new LinkedHashMap<>(); for (Aggregation subAgg : aggregation.getAggregations()) { - nested.put( - subAgg.getName(), - getExtractor(subAgg).value( - subAgg, - fieldTypeMap, - lookupFieldPrefix.isEmpty() ? agg.getName() : lookupFieldPrefix + "." + agg.getName() - ) - ); + nested.put(subAgg.getName(), getExtractor(subAgg).value(subAgg, fieldTypeMap, subAggLookupFieldPrefix)); } return nested; @@ -392,23 +387,17 @@ static class MultiBucketsAggExtractor implements AggValueExtractor { public Object value(Aggregation agg, Map fieldTypeMap, String lookupFieldPrefix) { MultiBucketsAggregation aggregation = (MultiBucketsAggregation) agg; - HashMap nested = new HashMap<>(); + var subAggLookupFieldPrefix = lookupFieldPrefix.isEmpty() ? agg.getName() : lookupFieldPrefix + "." + agg.getName(); + Map nested = Maps.newLinkedHashMapWithExpectedSize(aggregation.getBuckets().size()); for (MultiBucketsAggregation.Bucket bucket : aggregation.getBuckets()) { String bucketKey = bucketKeyTransfomer.apply(bucket.getKeyAsString()); if (bucket.getAggregations().iterator().hasNext() == false) { nested.put(bucketKey, bucket.getDocCount()); } else { - HashMap nestedBucketObject = new HashMap<>(); + Map nestedBucketObject = new LinkedHashMap<>(); for (Aggregation subAgg : bucket.getAggregations()) { - nestedBucketObject.put( - subAgg.getName(), - getExtractor(subAgg).value( - subAgg, - fieldTypeMap, - lookupFieldPrefix.isEmpty() ? agg.getName() : lookupFieldPrefix + "." + agg.getName() - ) - ); + nestedBucketObject.put(subAgg.getName(), getExtractor(subAgg).value(subAgg, fieldTypeMap, subAggLookupFieldPrefix)); } nested.put(bucketKey, nestedBucketObject); } @@ -441,18 +430,18 @@ public Object value(Aggregation agg, Map fieldTypeMap, String lo if (aggregation.bottomRight() == null || aggregation.topLeft() == null) { return null; } - final Map geoShape = new HashMap<>(); + final Map geoShape = new LinkedHashMap<>(); // If the two geo_points are equal, it is a point if (aggregation.topLeft().equals(aggregation.bottomRight())) { geoShape.put(FIELD_TYPE, POINT); - geoShape.put(FIELD_COORDINATES, Arrays.asList(aggregation.topLeft().getLon(), aggregation.bottomRight().getLat())); + geoShape.put(FIELD_COORDINATES, List.of(aggregation.topLeft().getLon(), aggregation.bottomRight().getLat())); // If only the lat or the lon of the two geo_points are equal, than we know it should be a line } else if (Double.compare(aggregation.topLeft().getLat(), aggregation.bottomRight().getLat()) == 0 || Double.compare(aggregation.topLeft().getLon(), aggregation.bottomRight().getLon()) == 0) { geoShape.put(FIELD_TYPE, LINESTRING); geoShape.put( FIELD_COORDINATES, - Arrays.asList( + List.of( new Double[] { aggregation.topLeft().getLon(), aggregation.topLeft().getLat() }, new Double[] { aggregation.bottomRight().getLon(), aggregation.bottomRight().getLat() } ) @@ -465,7 +454,7 @@ public Object value(Aggregation agg, Map fieldTypeMap, String lo geoShape.put( FIELD_COORDINATES, Collections.singletonList( - Arrays.asList( + List.of( new Double[] { tl.getLon(), tl.getLat() }, new Double[] { br.getLon(), tl.getLat() }, new Double[] { br.getLon(), br.getLat() }, @@ -495,12 +484,12 @@ static class GeoTileBucketKeyExtractor implements BucketKeyExtractor { public Object value(Object key, String type) { assert key instanceof String; Rectangle rectangle = GeoTileUtils.toBoundingBox(key.toString()); - final Map geoShape = new HashMap<>(); + final Map geoShape = Maps.newLinkedHashMapWithExpectedSize(2); geoShape.put(FIELD_TYPE, POLYGON); geoShape.put( FIELD_COORDINATES, Collections.singletonList( - Arrays.asList( + List.of( new Double[] { rectangle.getMaxLon(), rectangle.getMinLat() }, new Double[] { rectangle.getMinLon(), rectangle.getMinLat() }, new Double[] { rectangle.getMinLon(), rectangle.getMaxLat() }, diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java index a894dadc24afc..38f3223d840b2 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java @@ -33,6 +33,9 @@ import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; +import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; +import org.elasticsearch.xpack.core.transform.transforms.TransformState; import java.nio.file.Path; import java.util.ArrayList; @@ -44,6 +47,9 @@ import java.util.Map.Entry; import java.util.Set; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class TransformsCheckpointServiceTests extends ESTestCase { public void testExtractIndexCheckpoints() { @@ -114,6 +120,38 @@ public void testExtractIndexCheckpointsInconsistentGlobalCheckpoints() { } } + public void testTransformCheckpointingInfoWithZeroLastCheckpoint() { + var transformState = mock(TransformState.class); + when(transformState.getCheckpoint()).thenReturn(0L); + var position = mock(TransformIndexerPosition.class); + when(transformState.getPosition()).thenReturn(position); + var progress = mock(TransformProgress.class); + when(transformState.getProgress()).thenReturn(progress); + + var checkpointingInfo = TransformCheckpointService.deriveBasicCheckpointingInfo(transformState); + + assertEquals(checkpointingInfo.getLast().getCheckpoint(), 0L); + assertEquals(checkpointingInfo.getNext().getCheckpoint(), 0L); + assertSame(checkpointingInfo.getNext().getPosition(), position); + assertSame(checkpointingInfo.getNext().getCheckpointProgress(), progress); + } + + public void testTransformCheckpointingInfoWithNonZeroLastCheckpoint() { + var transformState = mock(TransformState.class); + when(transformState.getCheckpoint()).thenReturn(1L); + var position = mock(TransformIndexerPosition.class); + when(transformState.getPosition()).thenReturn(position); + var progress = mock(TransformProgress.class); + when(transformState.getProgress()).thenReturn(progress); + + var checkpointingInfo = TransformCheckpointService.deriveBasicCheckpointingInfo(transformState); + + assertEquals(checkpointingInfo.getLast().getCheckpoint(), 1L); + assertEquals(checkpointingInfo.getNext().getCheckpoint(), 2L); + assertSame(checkpointingInfo.getNext().getPosition(), position); + assertSame(checkpointingInfo.getNext().getCheckpointProgress(), progress); + } + /** * Create a random set of 3 index names * @return set of indices a simulated user has access to diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java index 12af48faf8e38..01eb0957501c3 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformTaskTests.java @@ -34,8 +34,12 @@ import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.core.transform.TransformConfigVersion; import org.elasticsearch.xpack.core.transform.transforms.AuthorizationState; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpointingInfo; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; +import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; @@ -44,7 +48,6 @@ import org.elasticsearch.xpack.transform.notifications.MockTransformAuditor; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; import org.elasticsearch.xpack.transform.persistence.InMemoryTransformConfigManager; -import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler; import org.junit.After; import org.junit.Before; @@ -61,7 +64,9 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -95,25 +100,6 @@ public void testStopOnFailedTaskWithStoppedIndexer() { TransformConfig transformConfig = TransformConfigTests.randomTransformConfigWithoutHeaders(); TransformAuditor auditor = MockTransformAuditor.createMockAuditor(); - TransformConfigManager transformsConfigManager = new InMemoryTransformConfigManager(); - TransformCheckpointService transformsCheckpointService = new TransformCheckpointService( - clock, - Settings.EMPTY, - new ClusterService( - Settings.EMPTY, - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - null, - (TaskManager) null - ), - transformsConfigManager, - auditor - ); - TransformServices transformServices = new TransformServices( - transformsConfigManager, - transformsCheckpointService, - auditor, - new TransformScheduler(clock, threadPool, Settings.EMPTY, TimeValue.ZERO) - ); TransformState transformState = new TransformState( TransformTaskState.FAILED, @@ -144,12 +130,7 @@ public void testStopOnFailedTaskWithStoppedIndexer() { transformTask.init(mock(PersistentTasksService.class), taskManager, "task-id", 42); - ClientTransformIndexerBuilder indexerBuilder = new ClientTransformIndexerBuilder(); - indexerBuilder.setClient(new ParentTaskAssigningClient(client, TaskId.EMPTY_TASK_ID)) - .setTransformConfig(transformConfig) - .setTransformServices(transformServices); - - transformTask.initializeIndexer(indexerBuilder); + transformTask.initializeIndexer(indexerBuilder(transformConfig, transformServices(clock, auditor, threadPool))); TransformState state = transformTask.getState(); assertEquals(TransformTaskState.FAILED, state.getTaskState()); assertEquals(IndexerState.STOPPED, state.getIndexerState()); @@ -186,6 +167,34 @@ public void testStopOnFailedTaskWithStoppedIndexer() { assertEquals(state.getReason(), null); } + private TransformServices transformServices(Clock clock, TransformAuditor auditor, ThreadPool threadPool) { + var transformsConfigManager = new InMemoryTransformConfigManager(); + var transformsCheckpointService = new TransformCheckpointService( + clock, + Settings.EMPTY, + new ClusterService( + Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + null, + (TaskManager) null + ), + transformsConfigManager, + auditor + ); + return new TransformServices( + transformsConfigManager, + transformsCheckpointService, + auditor, + new TransformScheduler(clock, threadPool, Settings.EMPTY, TimeValue.ZERO) + ); + } + + private ClientTransformIndexerBuilder indexerBuilder(TransformConfig transformConfig, TransformServices transformServices) { + return new ClientTransformIndexerBuilder().setClient(new ParentTaskAssigningClient(client, TaskId.EMPTY_TASK_ID)) + .setTransformConfig(transformConfig) + .setTransformServices(transformServices); + } + public void testStopOnFailedTaskWithoutIndexer() { ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.executor("generic")).thenReturn(mock(ExecutorService.class)); @@ -449,6 +458,71 @@ public void testApplyNewAuthState() { assertThat(transformTask.getContext().getAuthState(), is(nullValue())); } + public void testDeriveBasicCheckpointingInfoWithNoIndexer() { + var transformTask = createTransformTask( + TransformConfigTests.randomTransformConfigWithoutHeaders(), + MockTransformAuditor.createMockAuditor() + ); + var checkpointingInfo = transformTask.deriveBasicCheckpointingInfo(); + assertThat(checkpointingInfo, sameInstance(TransformCheckpointingInfo.EMPTY)); + } + + private TransformTask createTransformTask(TransformConfig transformConfig, MockTransformAuditor auditor) { + var threadPool = mock(ThreadPool.class); + + var transformState = new TransformState( + TransformTaskState.STARTED, + IndexerState.STARTED, + null, + 0L, + "because", + null, + null, + false, + null + ); + + return new TransformTask( + 42, + "some_type", + "some_action", + TaskId.EMPTY_TASK_ID, + createTransformTaskParams(transformConfig.getId()), + transformState, + new TransformScheduler(Clock.systemUTC(), threadPool, Settings.EMPTY, TimeValue.ZERO), + auditor, + threadPool, + Collections.emptyMap() + ); + } + + public void testDeriveBasicCheckpointingInfoWithIndexer() { + var lastCheckpoint = mock(TransformCheckpoint.class); + when(lastCheckpoint.getCheckpoint()).thenReturn(5L); + var nextCheckpoint = mock(TransformCheckpoint.class); + when(nextCheckpoint.getCheckpoint()).thenReturn(6L); + var position = mock(TransformIndexerPosition.class); + var progress = mock(TransformProgress.class); + + var transformConfig = TransformConfigTests.randomTransformConfigWithoutHeaders(); + var auditor = MockTransformAuditor.createMockAuditor(); + var transformTask = createTransformTask(transformConfig, auditor); + + transformTask.initializeIndexer( + indexerBuilder(transformConfig, transformServices(Clock.systemUTC(), auditor, threadPool)).setLastCheckpoint(lastCheckpoint) + .setNextCheckpoint(nextCheckpoint) + .setInitialPosition(position) + .setProgress(progress) + ); + + var checkpointingInfo = transformTask.deriveBasicCheckpointingInfo(); + assertThat(checkpointingInfo, not(sameInstance(TransformCheckpointingInfo.EMPTY))); + assertThat(checkpointingInfo.getLast().getCheckpoint(), equalTo(5L)); + assertThat(checkpointingInfo.getNext().getCheckpoint(), equalTo(6L)); + assertThat(checkpointingInfo.getNext().getPosition(), sameInstance(position)); + assertThat(checkpointingInfo.getNext().getCheckpointProgress(), sameInstance(progress)); + } + private static TransformTaskParams createTransformTaskParams(String transformId) { return new TransformTaskParams(transformId, TransformConfigVersion.CURRENT, TimeValue.timeValueSeconds(10), false); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java index 8d0bd4f9d8019..37e80f7459b2b 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java @@ -41,13 +41,18 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfig; import org.elasticsearch.xpack.transform.transforms.pivot.AggregationResultUtils.BucketKeyExtractor; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.equalTo; @@ -75,7 +80,7 @@ public String getWriteableName() { @Override public List getValuesAsStrings(String name) { - return List.of(values.get(name).toString()); + return List.of(values.get(name)); } @Override @@ -184,7 +189,7 @@ public void testExtractCompositeAggregationResults() throws IOException { asMap(targetField, "ID2", aggName, 28.99), asMap(targetField, "ID3", aggName, null) ); - Map fieldTypeMap = asStringMap(targetField, "keyword", aggName, "double"); + Map fieldTypeMap = Map.of(targetField, "keyword", aggName, "double"); executeTest(groupBy, aggregationBuilders, List.of(), input, fieldTypeMap, expected, 11); } @@ -207,7 +212,6 @@ public void testExtractCompositeAggregationResultsMultipleGroups() throws IOExce }""", targetField, targetField2)); String aggName = randomAlphaOfLengthBetween(5, 10); - String aggTypedName = "avg#" + aggName; List aggregationBuilders = List.of(AggregationBuilders.avg(aggName)); InternalComposite input = createComposite( @@ -241,7 +245,7 @@ public void testExtractCompositeAggregationResultsMultipleGroups() throws IOExce asMap(targetField, "ID2", targetField2, "ID1_2", aggName, 28.99), asMap(targetField, "ID3", targetField2, "ID2_2", aggName, null) ); - Map fieldTypeMap = asStringMap(aggName, "double", targetField, "keyword", targetField2, "keyword"); + Map fieldTypeMap = Map.of(aggName, "double", targetField, "keyword", targetField2, "keyword"); executeTest(groupBy, aggregationBuilders, List.of(), input, fieldTypeMap, expected, 6); } @@ -288,7 +292,7 @@ public void testExtractCompositeAggregationResultsMultiAggregations() throws IOE asMap(targetField, "ID2", aggName, 28.99, aggName2, 222.33), asMap(targetField, "ID3", aggName, 12.55, aggName2, null) ); - Map fieldTypeMap = asStringMap(targetField, "keyword", aggName, "double", aggName2, "double"); + Map fieldTypeMap = Map.of(targetField, "keyword", aggName, "double", aggName2, "double"); executeTest(groupBy, aggregationBuilders, List.of(), input, fieldTypeMap, expected, 200); } @@ -353,7 +357,7 @@ public void testExtractCompositeAggregationResultsMultiAggregationsAndTypes() th asMap(targetField, "ID2", targetField2, "ID1_2", aggName, 28.99, aggName2, "-2.44F"), asMap(targetField, "ID3", targetField2, "ID2_2", aggName, 12.55, aggName2, null) ); - Map fieldTypeMap = asStringMap( + Map fieldTypeMap = Map.of( aggName, "double", aggName2, @@ -419,7 +423,7 @@ public void testExtractCompositeAggregationResultsWithDynamicType() throws IOExc asMap(targetField, "ID2", targetField2, "ID1_2", aggName, asMap("field", 2.13)), asMap(targetField, "ID3", targetField2, "ID2_2", aggName, null) ); - Map fieldTypeMap = asStringMap(targetField, "keyword", targetField2, "keyword"); + Map fieldTypeMap = Map.of(targetField, "keyword", targetField2, "keyword"); executeTest(groupBy, aggregationBuilders, List.of(), input, fieldTypeMap, expected, 6); } @@ -488,7 +492,7 @@ public void testExtractCompositeAggregationResultsWithPipelineAggregation() thro asMap(targetField, "ID2", targetField2, "ID1_2", aggName, 2.13, pipelineAggName, 2.13), asMap(targetField, "ID3", targetField2, "ID2_2", aggName, 12.0, pipelineAggName, null) ); - Map fieldTypeMap = asStringMap(targetField, "keyword", targetField2, "keyword", aggName, "double"); + Map fieldTypeMap = Map.of(targetField, "keyword", targetField2, "keyword", aggName, "double"); executeTest(groupBy, aggregationBuilders, pipelineAggregationBuilders, input, fieldTypeMap, expected, 10); } @@ -566,7 +570,7 @@ public void testExtractCompositeAggregationResultsDocIDs() throws IOException { TransformIndexerStats stats = new TransformIndexerStats(); TransformProgress progress = new TransformProgress(); - Map fieldTypeMap = asStringMap(aggName, "double", targetField, "keyword", targetField2, "keyword"); + Map fieldTypeMap = Map.of(aggName, "double", targetField, "keyword", targetField2, "keyword"); List> resultFirstRun = runExtraction( groupBy, @@ -680,24 +684,21 @@ public void testSingleValueAggExtractor() { public void testMultiValueAggExtractor() { Aggregation agg = new TestMultiValueAggregation("mv_metric", Map.of("ip", "192.168.1.1")); - assertThat( AggregationResultUtils.getExtractor(agg).value(agg, Map.of("mv_metric.ip", "ip"), ""), equalTo(Map.of("ip", "192.168.1.1")) ); agg = new TestMultiValueAggregation("mv_metric", Map.of("top_answer", "fortytwo")); - assertThat( AggregationResultUtils.getExtractor(agg).value(agg, Map.of("mv_metric.written_answer", "written_answer"), ""), equalTo(Map.of("top_answer", "fortytwo")) ); - agg = new TestMultiValueAggregation("mv_metric", Map.of("ip", "192.168.1.1", "top_answer", "fortytwo")); - + agg = new TestMultiValueAggregation("mv_metric", asOrderedMap("ip", "192.168.1.1", "top_answer", "fortytwo")); assertThat( AggregationResultUtils.getExtractor(agg).value(agg, Map.of("mv_metric.top_answer", "keyword", "mv_metric.ip", "ip"), ""), - equalTo(Map.of("top_answer", "fortytwo", "ip", "192.168.1.1")) + hasEqualEntriesInOrder(asOrderedMap("ip", "192.168.1.1", "top_answer", "fortytwo")) ); } @@ -715,22 +716,19 @@ public void testNumericMultiValueAggExtractor() { AggregationResultUtils.getExtractor(agg).value(agg, Map.of("mv_metric.exact_answer", "long"), ""), equalTo(Map.of("exact_answer", 42L)) ); - agg = new TestNumericMultiValueAggregation( "mv_metric", - Map.of("approx_answer", Double.valueOf(42.2), "exact_answer", Double.valueOf(42.0)) + asOrderedMap("approx_answer", Double.valueOf(42.2), "exact_answer", Double.valueOf(42.0)) ); - assertThat( AggregationResultUtils.getExtractor(agg) .value(agg, Map.of("mv_metric.approx_answer", "double", "mv_metric.exact_answer", "long"), ""), - equalTo(Map.of("approx_answer", Double.valueOf(42.2), "exact_answer", Long.valueOf(42))) + hasEqualEntriesInOrder(asOrderedMap("approx_answer", Double.valueOf(42.2), "exact_answer", Long.valueOf(42))) ); - assertThat( AggregationResultUtils.getExtractor(agg) .value(agg, Map.of("filter.mv_metric.approx_answer", "double", "filter.mv_metric.exact_answer", "long"), "filter"), - equalTo(Map.of("approx_answer", 42.2, "exact_answer", Long.valueOf(42))) + hasEqualEntriesInOrder(asOrderedMap("approx_answer", 42.2, "exact_answer", Long.valueOf(42))) ); } @@ -791,13 +789,13 @@ public void testGeoBoundsAggExtractor() { String type = "point"; for (int i = 0; i < numberOfRuns; i++) { - Map expectedObject = new HashMap<>(); - expectedObject.put("type", type); double lat = randomDoubleBetween(-90.0, 90.0, false); double lon = randomDoubleBetween(-180.0, 180.0, false); - expectedObject.put("coordinates", List.of(lon, lat)); agg = createGeoBounds(new GeoPoint(lat, lon), new GeoPoint(lat, lon)); - assertThat(AggregationResultUtils.getExtractor(agg).value(agg, Map.of(), ""), equalTo(expectedObject)); + assertThat( + AggregationResultUtils.getExtractor(agg).value(agg, Map.of(), ""), + hasEqualEntriesInOrder(asOrderedMap("type", type, "coordinates", List.of(lon, lat))) + ); } type = "linestring"; @@ -895,13 +893,16 @@ public void testPercentilesAggExtractor() { ); assertThat( AggregationResultUtils.getExtractor(agg).value(agg, Map.of(), ""), - equalTo(asMap("1", 0.0, "50", 22.2, "99", 43.3, "99_5", 100.3)) + hasEqualEntriesInOrder(asOrderedMap("1", 0.0, "50", 22.2, "99", 43.3, "99_5", 100.3)) ); } public void testPercentilesAggExtractorNaN() { Aggregation agg = createPercentilesAgg("p_agg", List.of(new Percentile(1, Double.NaN), new Percentile(50, Double.NaN))); - assertThat(AggregationResultUtils.getExtractor(agg).value(agg, Map.of(), ""), equalTo(asMap("1", null, "50", null))); + assertThat( + AggregationResultUtils.getExtractor(agg).value(agg, Map.of(), ""), + hasEqualEntriesInOrder(asOrderedMap("1", null, "50", null)) + ); } @SuppressWarnings("unchecked") @@ -928,8 +929,8 @@ public void testRangeAggExtractor() { ); assertThat( AggregationResultUtils.getExtractor(agg).value(agg, Map.of(), ""), - equalTo( - asMap( + hasEqualEntriesInOrder( + asOrderedMap( "*-10_5", 10L, "10_5-19_5", @@ -951,6 +952,56 @@ public void testRangeAggExtractor() { ); } + private static Matcher hasEqualEntriesInOrder(Map expected) { + return new BaseMatcher() { + @Override + public boolean matches(Object o) { + if (o instanceof Map) { + return matches((Map) o); + } + return false; + } + + public boolean matches(Map o) { + var expectedEntries = expected.entrySet().iterator(); + var actualEntries = o.entrySet().iterator(); + while (expectedEntries.hasNext() && actualEntries.hasNext()) { + var expectedEntry = expectedEntries.next(); + var actualEntry = actualEntries.next(); + assertThat( + "Entry is out of order. Expected order: " + + mapToString(expected, expectedEntry) + + ", Actual order: " + + mapToString(o, actualEntry), + actualEntry, + equalTo(expectedEntry) + ); + } + return expectedEntries.hasNext() == false && actualEntries.hasNext() == false; + } + + private String mapToString(Map map, Object node) { + return map.entrySet().stream().map(entry -> { + var entryAsString = entry.getKey() + "=" + entry.getValue(); + if (node == entry) { + return "<<" + entryAsString + ">>"; + } + return entryAsString; + }).collect(Collectors.joining(", ", "{", "}")); + } + + @Override + public void describeTo(Description description) { + description.appendText( + expected.entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining(", ", "{", "}")) + ); + } + }; + } + public static InternalSingleBucketAggregation createSingleBucketAgg( String name, long docCount, @@ -982,8 +1033,8 @@ public void testSingleBucketAggExtractor() { createSingleMetricAgg("sub2", 33.33, "thirty_three") ); assertThat( - AggregationResultUtils.getExtractor(agg).value(agg, asStringMap("sba2.sub1", "long", "sba2.sub2", "float"), ""), - equalTo(asMap("sub1", 100L, "sub2", 33.33)) + AggregationResultUtils.getExtractor(agg).value(agg, Map.of("sba2.sub1", "long", "sba2.sub2", "float"), ""), + hasEqualEntriesInOrder(asOrderedMap("sub1", 100L, "sub2", 33.33)) ); agg = createSingleBucketAgg( @@ -994,8 +1045,8 @@ public void testSingleBucketAggExtractor() { createSingleBucketAgg("sub3", 42L) ); assertThat( - AggregationResultUtils.getExtractor(agg).value(agg, asStringMap("sba3.sub1", "long", "sba3.sub2", "double"), ""), - equalTo(asMap("sub1", 100L, "sub2", 33.33, "sub3", 42L)) + AggregationResultUtils.getExtractor(agg).value(agg, Map.of("sba3.sub1", "long", "sba3.sub2", "double"), ""), + hasEqualEntriesInOrder(asOrderedMap("sub1", 100L, "sub2", 33.33, "sub3", 42L)) ); agg = createSingleBucketAgg( @@ -1007,8 +1058,8 @@ public void testSingleBucketAggExtractor() { ); assertThat( AggregationResultUtils.getExtractor(agg) - .value(agg, asStringMap("sba4.sub3.subsub1", "double", "sba4.sub2", "float", "sba4.sub1", "long"), ""), - equalTo(asMap("sub1", 100L, "sub2", 33.33, "sub3", asMap("subsub1", 11.1))) + .value(agg, Map.of("sba4.sub3.subsub1", "double", "sba4.sub2", "float", "sba4.sub1", "long"), ""), + hasEqualEntriesInOrder(asOrderedMap("sub1", 100L, "sub2", 33.33, "sub3", asMap("subsub1", 11.1))) ); } @@ -1094,22 +1145,27 @@ private GroupConfig parseGroupConfig(String json) throws IOException { } static Map asMap(Object... fields) { + return asMap(HashMap::new, fields); + } + + static Map asOrderedMap(Object... fields) { + return asMap(LinkedHashMap::new, fields); + } + + static Map asMap(Supplier> mapFactory, Object... fields) { assert fields.length % 2 == 0; - final Map map = new HashMap<>(); + var map = mapFactory.get(); for (int i = 0; i < fields.length; i += 2) { - String field = (String) fields[i]; + var field = (String) fields[i]; map.put(field, fields[i + 1]); } return map; } - static Map asStringMap(String... strings) { - assert strings.length % 2 == 0; - final Map map = new HashMap<>(); - for (int i = 0; i < strings.length; i += 2) { - String field = strings[i]; - map.put(field, strings[i + 1]); - } + static Map asOrderedMap(K k1, V v1, K k2, V v2) { + var map = new LinkedHashMap(); + map.put(k1, v1); + map.put(k2, v2); return map; } } diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java index 7f6e645b15015..590f0816d4aa7 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -22,6 +23,7 @@ import org.elasticsearch.xpack.vectortile.rest.RestVectorTileAction; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; public class VectorTilePlugin extends Plugin implements ActionPlugin { @@ -40,7 +42,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new RestVectorTileAction()); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index f107bac568902..4af03808b347a 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -38,6 +38,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexModule; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; @@ -203,6 +204,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -697,7 +699,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { if (false == enabled) { return emptyList(); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java index 1d93e999a4407..354190c907efe 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java @@ -44,7 +44,7 @@ public void testWatcherDisabledTests() throws Exception { List> executorBuilders = watcher.getExecutorBuilders(settings); assertThat(executorBuilders, hasSize(0)); assertThat(watcher.getActions(), hasSize(2)); - assertThat(watcher.getRestHandlers(settings, null, null, null, null, null, null, null), hasSize(0)); + assertThat(watcher.getRestHandlers(settings, null, null, null, null, null, null, null, null), hasSize(0)); // ensure index module is not called, even if watches index is tried IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(Watch.INDEX, settings); diff --git a/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java b/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java index 15336286cc2fc..4bd88217a2d25 100644 --- a/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java +++ b/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; @@ -30,6 +31,7 @@ import java.io.IOException; import java.util.List; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -49,7 +51,8 @@ public List getRestHandlers( IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster + Supplier nodesInCluster, + Predicate clusterSupportsFeature ) { return List.of(new FreezeIndexRestEndpoint()); } diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle index 54b455d483b9a..87db264356484 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle @@ -8,7 +8,7 @@ apply plugin: 'elasticsearch.rest-resources' restResources { restApi { include '_common', 'bulk', 'field_caps', 'security', 'search', 'clear_scroll', 'scroll', 'async_search', 'cluster', - 'indices', 'open_point_in_time', 'close_point_in_time', 'terms_enum', 'esql' + 'indices', 'open_point_in_time', 'close_point_in_time', 'terms_enum', 'esql', 'enrich' } } @@ -25,6 +25,8 @@ def fulfillingCluster = testClusters.register('fulfilling-cluster') { module ':x-pack:plugin:async-search' module ':x-pack:plugin:ql' module ':x-pack:plugin:esql' + module ':modules:ingest-common' + module ':x-pack:plugin:enrich' user username: "test_user", password: "x-pack-test-password" } @@ -38,6 +40,8 @@ def queryingCluster = testClusters.register('querying-cluster') { module ':x-pack:plugin:async-search' module ':x-pack:plugin:ql' module ':x-pack:plugin:esql' + module ':modules:ingest-common' + module ':x-pack:plugin:enrich' setting 'cluster.remote.connections_per_cluster', "1" user username: "test_user", password: "x-pack-test-password" diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml index 36002f3cde470..f3861264d6056 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/fulfilling_cluster/10_basic.yml @@ -457,3 +457,43 @@ setup: - '{"since" : "2023-01-04", "cost": 100, "tag": "headphone"}' - '{"index": {"_index": "esql_index"}}' - '{"since" : "2023-01-05", "cost": 20, "tag": "headphone"}' + - do: + indices.create: + index: suggestions + body: + mappings: + properties: + tag: + type: keyword + phrase: + type: keyword + - do: + bulk: + index: "suggestions" + refresh: true + body: + - { "index": { } } + - { "tag": "laptop", "phrase": "the best battery life laptop" } + - { "index": { } } + - { "tag": "computer", "phrase": "best desktop for programming" } + - { "index": { } } + - { "tag": "monitor", "phrase": "4k or 5k or 6K monitor?" } + - { "index": { } } + - { "tag": "headphone", "phrase": "the best noise-cancelling headphones" } + - { "index": { } } + - { "tag": "tablet", "phrase": "tablets for kids" } + - do: + enrich.put_policy: + name: suggestions + body: + match: + indices: [ "suggestions" ] + match_field: "tag" + enrich_fields: [ "phrase" ] + - do: + enrich.execute_policy: + name: suggestions + - do: + indices.delete: + index: suggestions + diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/100_resolve_index.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/100_resolve_index.yml index b9dbb0a070af4..883244624881d 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/100_resolve_index.yml +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/100_resolve_index.yml @@ -21,38 +21,41 @@ - match: {indices.2.attributes.0: hidden} - match: {indices.2.attributes.1: open} - match: {indices.2.data_stream: simple-data-stream2} - - match: {indices.3.name: my_remote_cluster:.security-7} - - match: {indices.3.attributes.0: hidden} - - match: {indices.4.name: my_remote_cluster:closed_index} - - match: {indices.4.aliases.0: aliased_closed_index} - - match: {indices.4.attributes.0: closed} - - match: {indices.5.name: my_remote_cluster:esql_index } - - match: {indices.5.attributes.0: open } - - match: {indices.6.name: my_remote_cluster:field_caps_index_1} - - match: {indices.6.attributes.0: open} - - match: {indices.7.name: my_remote_cluster:field_caps_index_3} + - match: {indices.3.attributes.0: hidden } + - match: {indices.4.name: my_remote_cluster:.security-7} + - match: {indices.4.attributes.0: hidden} + - match: {indices.5.name: my_remote_cluster:closed_index} + - match: {indices.5.aliases.0: aliased_closed_index} + - match: {indices.5.attributes.0: closed} + - match: {indices.6.name: my_remote_cluster:esql_index } + - match: {indices.6.attributes.0: open } + - match: {indices.7.name: my_remote_cluster:field_caps_index_1} - match: {indices.7.attributes.0: open} - - match: {indices.8.name: my_remote_cluster:point_in_time_index } - - match: {indices.8.attributes.0: open } - - match: {indices.9.name: my_remote_cluster:secured_via_alias} - - match: {indices.9.attributes.0: open} - - match: {indices.10.name: my_remote_cluster:shared_index} + - match: {indices.8.name: my_remote_cluster:field_caps_index_3} + - match: {indices.8.attributes.0: open} + - match: {indices.9.name: my_remote_cluster:point_in_time_index } + - match: {indices.9.attributes.0: open } + - match: {indices.10.name: my_remote_cluster:secured_via_alias} - match: {indices.10.attributes.0: open} - - match: {indices.11.name: my_remote_cluster:single_doc_index} + - match: {indices.11.name: my_remote_cluster:shared_index} - match: {indices.11.attributes.0: open} - - match: {indices.12.name: my_remote_cluster:terms_enum_index } - - match: {indices.12.attributes.0: open } - - match: {indices.13.name: my_remote_cluster:test_index} - - match: {indices.13.aliases.0: aliased_test_index} - - match: {indices.13.attributes.0: open} - - match: {aliases.0.name: my_remote_cluster:.security} - - match: {aliases.0.indices.0: .security-7} - - match: {aliases.1.name: my_remote_cluster:aliased_closed_index} - - match: {aliases.1.indices.0: closed_index} - - match: {aliases.2.name: my_remote_cluster:aliased_test_index} - - match: {aliases.2.indices.0: test_index} - - match: {aliases.3.name: my_remote_cluster:secure_alias} - - match: {aliases.3.indices.0: secured_via_alias} + - match: {indices.12.name: my_remote_cluster:single_doc_index} + - match: {indices.12.attributes.0: open} + - match: {indices.13.name: my_remote_cluster:terms_enum_index } + - match: {indices.13.attributes.0: open } + - match: {indices.14.name: my_remote_cluster:test_index} + - match: {indices.14.aliases.0: aliased_test_index} + - match: {indices.14.attributes.0: open} + - match: {aliases.0.name: my_remote_cluster:.enrich-suggestions} + - match: {aliases.0.indices.0: "/.enrich-suggestions-\\d+/"} + - match: {aliases.1.name: my_remote_cluster:.security } + - match: {aliases.1.indices.0: .security-7 } + - match: {aliases.2.name: my_remote_cluster:aliased_closed_index} + - match: {aliases.2.indices.0: closed_index} + - match: {aliases.3.name: my_remote_cluster:aliased_test_index} + - match: {aliases.3.indices.0: test_index} + - match: {aliases.4.name: my_remote_cluster:secure_alias} + - match: {aliases.4.indices.0: secured_via_alias} - match: {data_streams.0.name: my_remote_cluster:simple-data-stream1} - match: {data_streams.0.backing_indices.0: "/\\.ds-simple-data-stream1-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/"} - match: {data_streams.0.timestamp_field: "@timestamp"} diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml index 1894a26e80f33..30c30365276f4 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml @@ -19,7 +19,7 @@ setup: name: "x_cluster_role" body: > { - "cluster": [], + "cluster": [ "monitor_enrich" ], "indices": [ { "names": ["local_index", "esql_local"], @@ -42,19 +42,6 @@ setup: body: > { } ---- -teardown: - - do: - security.delete_user: - username: "joe" - ignore: 404 - - do: - security.delete_role: - name: "x_cluster_role" - ignore: 404 - ---- -"Index data and search on the mixed cluster": - do: indices.create: @@ -84,6 +71,23 @@ teardown: - '{"index": {"_index": "esql_local"}}' - '{"since" : "2023-01-05", "cost": 50, "tag": "headphone"}' +--- +teardown: + - do: + indices.delete: + index: esql_local + - do: + security.delete_user: + username: "joe" + ignore: 404 + - do: + security.delete_role: + name: "x_cluster_role" + ignore: 404 + +--- +"Index data and search on the mixed cluster": + - do: headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } esql.query: @@ -138,6 +142,74 @@ teardown: - match: {values.3.1: "laptop" } - match: {values.3.2: 2100 } +--- +"Enrich across clusters": + - skip: + version: " - 8.12.99" + reason: "Enrich across clusters available in 8.13 or later" + - do: + indices.create: + index: suggestions + body: + mappings: + properties: + tag: + type: keyword + phrase: + type: keyword + - do: + bulk: + index: "suggestions" + refresh: true + body: + - { "index": { } } + - { "tag": "laptop", "phrase": "the best battery life laptop" } + - { "index": { } } + - { "tag": "computer", "phrase": "best desktop for programming" } + - { "index": { } } + - { "tag": "monitor", "phrase": "4k or 5k or 6K monitor?" } + - { "index": { } } + - { "tag": "headphone", "phrase": "the best noise-cancelling headphones" } + - { "index": { } } + - { "tag": "tablet", "phrase": "tablets for kids" } + - do: + enrich.put_policy: + name: suggestions + body: + match: + indices: [ "suggestions" ] + match_field: "tag" + enrich_fields: [ "phrase" ] + - do: + enrich.execute_policy: + name: suggestions - do: indices.delete: - index: esql_local + index: suggestions + + - do: + headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } + esql.query: + body: + query: 'FROM *:esql*,esql_* | STATS total = sum(cost) by tag | SORT total DESC | LIMIT 3 | ENRICH suggestions | KEEP tag, total, phrase' + + - match: {columns.0.name: "tag"} + - match: {columns.0.type: "keyword"} + - match: {columns.1.name: "total" } + - match: {columns.1.type: "long" } + - match: {columns.2.name: "phrase" } + - match: {columns.2.type: "keyword" } + + - match: {values.0.0: "computer"} + - match: {values.0.1: 2200} + - match: {values.0.2: "best desktop for programming"} + - match: {values.1.0: "laptop"} + - match: {values.1.1: 2100 } + - match: {values.1.2: "the best battery life laptop"} + - match: {values.2.0: "monitor" } + - match: {values.2.1: 1000 } + - match: {values.2.2: "4k or 5k or 6K monitor?" } + + - do: + enrich.delete_policy: + name: suggestions