diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index 221dccae91322..d4bb0c1189e8d 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -168,7 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { vmParameters = [ '-ea', '-Djava.security.manager=allow', - '-Djava.locale.providers=SPI,CLDR', + '-Djava.locale.providers=CLDR', '-Des.nativelibs.path="' + testLibraryPath + '"', // TODO: only open these for mockito when it is modularized '--add-opens=java.base/java.security.cert=ALL-UNNAMED', diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 96cfa24c4c7d9..19ab49a851907 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -93,7 +93,7 @@ public void execute(Task t) { mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile()); // TODO remove once jvm.options are added to test system properties - test.systemProperty("java.locale.providers", "SPI,CLDR"); + test.systemProperty("java.locale.providers", "CLDR"); } }); test.getJvmArgumentProviders().add(nonInputProperties); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java index f7d2b26e069e1..bbf411dbf04fa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java @@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider { "org.elasticsearch.nativeaccess", "org.elasticsearch.plugin", "org.elasticsearch.plugin.analysis", - "org.elasticsearch.pluginclassloader", "org.elasticsearch.securesm", "org.elasticsearch.server", "org.elasticsearch.simdvec", diff --git a/distribution/tools/entitlement-agent/README.md b/distribution/tools/entitlement-agent/README.md new file mode 100644 index 0000000000000..ff40651706a7f --- /dev/null +++ b/distribution/tools/entitlement-agent/README.md @@ -0,0 +1,10 @@ +### Entitlement Agent + +This is a java agent that instruments sensitive class library methods with calls into the `entitlement-runtime` module to check for permissions granted under the _entitlements_ system. + +The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal. +With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers. + +This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the `entitlement-runtime` module. + +It is not responsible for permission granting or checking logic. That responsibility lies with `entitlement-runtime`. diff --git a/distribution/tools/entitlement-agent/build.gradle b/distribution/tools/entitlement-agent/build.gradle new file mode 100644 index 0000000000000..56e2ffac53fd7 --- /dev/null +++ b/distribution/tools/entitlement-agent/build.gradle @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.build' + +configurations { + entitlementRuntime +} + +dependencies { + entitlementRuntime project(":libs:elasticsearch-entitlement-runtime") + implementation project(":libs:elasticsearch-entitlement-runtime") + testImplementation project(":test:framework") +} + +tasks.named('test').configure { + dependsOn('jar') + jvmArgs "-javaagent:${ tasks.named('jar').flatMap{ it.archiveFile }.get()}" +} + +tasks.named('jar').configure { + manifest { + attributes( + 'Premain-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent' + , 'Can-Retransform-Classes': 'true' + ) + } +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + diff --git a/libs/plugin-classloader/src/main/java/module-info.java b/distribution/tools/entitlement-agent/src/main/java/module-info.java similarity index 78% rename from libs/plugin-classloader/src/main/java/module-info.java rename to distribution/tools/entitlement-agent/src/main/java/module-info.java index 90549c5c4a01b..df6fc154fc67f 100644 --- a/libs/plugin-classloader/src/main/java/module-info.java +++ b/distribution/tools/entitlement-agent/src/main/java/module-info.java @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -module org.elasticsearch.pluginclassloader { - exports org.elasticsearch.plugins.loader; +module org.elasticsearch.entitlement.agent { + requires java.instrument; + requires org.elasticsearch.entitlement.runtime; } diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java b/distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java similarity index 53% rename from server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java rename to distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java index 1a9618dd59f4d..b843e42f4a03e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java +++ b/distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java @@ -7,22 +7,15 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.cluster.ack; +package org.elasticsearch.entitlement.agent; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.entitlement.runtime.api.EntitlementChecks; -/** - * Identifies a cluster state update request with acknowledgement support - */ -public interface AckedRequest { +import java.lang.instrument.Instrumentation; - /** - * Returns the acknowledgement timeout - */ - TimeValue ackTimeout(); +public class EntitlementAgent { - /** - * Returns the timeout for the request to be completed on the master node - */ - TimeValue masterNodeTimeout(); + public static void premain(String agentArgs, Instrumentation inst) throws Exception { + EntitlementChecks.setAgentBooted(); + } } diff --git a/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java new file mode 100644 index 0000000000000..3927465570c98 --- /dev/null +++ b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.agent; + +import org.elasticsearch.entitlement.runtime.api.EntitlementChecks; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutSecurityManager; + +/** + * This is an end-to-end test that runs with the javaagent installed. + * It should exhaustively test every instrumented method to make sure it passes with the entitlement + * and fails without it. + * See {@code build.gradle} for how we set the command line arguments for this test. + */ +@WithoutSecurityManager +public class EntitlementAgentTests extends ESTestCase { + + public void testAgentBooted() { + assertTrue(EntitlementChecks.isAgentBooted()); + } + +} diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 784ecb18f8bd9..9832501073815 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.core.UpdateForV9; import java.util.List; import java.util.Map; @@ -61,7 +60,7 @@ static List systemJvmOptions(Settings nodeSettings, final Map systemJvmOptions(Settings nodeSettings, final Map e.isEmpty() == false).collect(Collectors.toList()); } - @UpdateForV9 // only use CLDR in v9+ - private static String getLocaleProviders() { - /* - * Specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday. - * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can. - * When on JDK 23+, use the default CLDR locale database, as COMPAT was removed in JDK 23. - */ - return Runtime.version().feature() >= 23 ? "SPI,CLDR" : "SPI,COMPAT"; - } - /* * The virtual file /proc/self/cgroup should list the current cgroup * membership. For each hierarchy, you can follow the cgroup path from diff --git a/docs/changelog/112645.yaml b/docs/changelog/112645.yaml new file mode 100644 index 0000000000000..cf4ef4609a1f3 --- /dev/null +++ b/docs/changelog/112645.yaml @@ -0,0 +1,6 @@ +pr: 112645 +summary: Add support for multi-value dimensions +area: Mapping +type: enhancement +issues: + - 110387 diff --git a/docs/changelog/112768.yaml b/docs/changelog/112768.yaml new file mode 100644 index 0000000000000..13d5b8eaae38f --- /dev/null +++ b/docs/changelog/112768.yaml @@ -0,0 +1,5 @@ +pr: 112768 +summary: Deduplicate Kuromoji User Dictionary +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/113102.yaml b/docs/changelog/113102.yaml new file mode 100644 index 0000000000000..ea9022e634caf --- /dev/null +++ b/docs/changelog/113102.yaml @@ -0,0 +1,5 @@ +pr: 113102 +summary: Trigger merges after recovery +area: Recovery +type: enhancement +issues: [] diff --git a/docs/changelog/113103.yaml b/docs/changelog/113103.yaml new file mode 100644 index 0000000000000..2ed98e0907bae --- /dev/null +++ b/docs/changelog/113103.yaml @@ -0,0 +1,6 @@ +pr: 113103 +summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological" +area: ES|QL +type: bug +issues: + - 112482 diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index b1d1d5a751057..e8380ce0aca17 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -133,6 +133,11 @@ unknown words. It can be set to: Whether punctuation should be discarded from the output. Defaults to `true`. +`lenient`:: + + Whether the `user_dictionary` should be deduplicated on the provided `text`. + False by default causing duplicates to generate an error. + `user_dictionary`:: + -- @@ -221,7 +226,8 @@ PUT kuromoji_sample "type": "kuromoji_tokenizer", "mode": "extended", "discard_punctuation": "false", - "user_dictionary": "userdict_ja.txt" + "user_dictionary": "userdict_ja.txt", + "lenient": "true" } }, "analyzer": { diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc index 1a3153fa3bea5..e7855f94758e1 100644 --- a/docs/plugins/analysis-nori.asciidoc +++ b/docs/plugins/analysis-nori.asciidoc @@ -58,6 +58,11 @@ It can be set to: Whether punctuation should be discarded from the output. Defaults to `true`. +`lenient`:: + + Whether the `user_dictionary` should be deduplicated on the provided `text`. + False by default causing duplicates to generate an error. + `user_dictionary`:: + -- @@ -104,7 +109,8 @@ PUT nori_sample "type": "nori_tokenizer", "decompound_mode": "mixed", "discard_punctuation": "false", - "user_dictionary": "userdict_ko.txt" + "user_dictionary": "userdict_ko.txt", + "lenient": "true" } }, "analyzer": { @@ -299,7 +305,6 @@ Which responds with: } -------------------------------------------------- - [[analysis-nori-speech]] ==== `nori_part_of_speech` token filter diff --git a/docs/reference/esql/functions/examples/date_diff.asciidoc b/docs/reference/esql/functions/examples/date_diff.asciidoc index f85bdf480c1c3..f75add7b80501 100644 --- a/docs/reference/esql/functions/examples/date_diff.asciidoc +++ b/docs/reference/esql/functions/examples/date_diff.asciidoc @@ -1,6 +1,6 @@ // This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. -*Example* +*Examples* [source.merge.styled,esql] ---- @@ -10,4 +10,15 @@ include::{esql-specs}/date.csv-spec[tag=docsDateDiff] |=== include::{esql-specs}/date.csv-spec[tag=docsDateDiff-result] |=== +When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted. +To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly. + +[source.merge.styled,esql] +---- +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs-result] +|=== diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json index 4ba0d598a689c..f4c4de53f72a3 100644 --- a/docs/reference/esql/functions/kibana/definition/date_diff.json +++ b/docs/reference/esql/functions/kibana/definition/date_diff.json @@ -54,7 +54,8 @@ } ], "examples" : [ - "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)" + "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)", + "ROW end_23=\"2023-12-31T23:59:59.999Z\"::DATETIME,\n start_24=\"2024-01-01T00:00:00.000Z\"::DATETIME,\n end_24=\"2024-12-31T23:59:59.999\"::DATETIME\n| EVAL end23_to_start24=DATE_DIFF(\"year\", end_23, start_24)\n| EVAL end23_to_end24=DATE_DIFF(\"year\", end_23, end_24)\n| EVAL start_to_end_24=DATE_DIFF(\"year\", start_24, end_24)" ], "preview" : false } diff --git a/docs/reference/esql/functions/kibana/docs/mv_avg.md b/docs/reference/esql/functions/kibana/docs/mv_avg.md index c3d7e5423f724..c5163f36129bf 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_avg.md +++ b/docs/reference/esql/functions/kibana/docs/mv_avg.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_AVG -Converts a multivalued field into a single valued field containing the average of all of the values. +Converts a multivalued field into a single valued field containing the average of all the values. ``` ROW a=[3, 5, 1, 6] diff --git a/docs/reference/esql/functions/kibana/docs/mv_sum.md b/docs/reference/esql/functions/kibana/docs/mv_sum.md index 16285d3c7229b..987017b34b743 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_sum.md +++ b/docs/reference/esql/functions/kibana/docs/mv_sum.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_SUM -Converts a multivalued field into a single valued field containing the sum of all of the values. +Converts a multivalued field into a single valued field containing the sum of all the values. ``` ROW a=[3, 5, 6] diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index 572cad591fba6..efa0c78b8356f 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -179,6 +179,7 @@ PUT _inference/text_embedding/my-e5-model "min_number_of_allocations": 3, "max_number_of_allocations": 10 }, + "num_threads": 1, "model_id": ".multilingual-e5-small" } } diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc index fdce94901984b..c7217f38d459b 100644 --- a/docs/reference/inference/service-elser.asciidoc +++ b/docs/reference/inference/service-elser.asciidoc @@ -147,7 +147,8 @@ PUT _inference/sparse_embedding/my-elser-model "enabled": true, "min_number_of_allocations": 3, "max_number_of_allocations": 10 - } + }, + "num_threads": 1 } } ------------------------------------------------------------ diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 59d307c4df0ad..a4be7026dffcd 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -163,7 +163,6 @@ index setting limits the number of dimensions in an index. Dimension fields have the following constraints: * The `doc_values` and `index` mapping parameters must be `true`. -* Field values cannot be an <>. // end::dimension[] * Dimension values are used to identify a document’s time series. If dimension values are altered in any way during indexing, the document will be stored as belonging to different from intended time series. As a result there are additional constraints: ** The field cannot use a <>. diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc index 08dd7ab7f4470..399cf29d4dd12 100644 --- a/docs/reference/query-dsl/sparse-vector-query.asciidoc +++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc @@ -104,7 +104,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index 8faecad1dbdb9..235a413df686f 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -68,7 +68,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/weighted-tokens-query.asciidoc b/docs/reference/query-dsl/weighted-tokens-query.asciidoc index d4318665a9778..fb051f4229df6 100644 --- a/docs/reference/query-dsl/weighted-tokens-query.asciidoc +++ b/docs/reference/query-dsl/weighted-tokens-query.asciidoc @@ -58,7 +58,7 @@ This value must between 1 and 100. Default: `5`. `tokens_weight_threshold`:: -(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index c19f4f7cf989b..80c86c7079f0c 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -35,6 +35,7 @@ can be configured using the https://www.elastic.co/guide/en/elasticsearch/refere ** These indices have many conflicting field mappings ** Many of those fields are included in the request These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). +In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. + To work around this issue, you have a number of options: ** Downgrade to an earlier version @@ -43,6 +44,14 @@ To work around this issue, you have a number of options: <> ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. +* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or +partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. +There is no workaround and the only option to is to upgrade to 8.15.2 when released. ++ +If you use synthetic source then you may be affected by this bug if the following is true: +** If you have more fields then the `index.mapping.total_fields.limit` setting allows. +** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. + [[breaking-8.15.0]] [float] === Breaking changes diff --git a/docs/reference/release-notes/8.15.1.asciidoc b/docs/reference/release-notes/8.15.1.asciidoc index 2c126cccbda9e..7c48f457e3b4e 100644 --- a/docs/reference/release-notes/8.15.1.asciidoc +++ b/docs/reference/release-notes/8.15.1.asciidoc @@ -15,6 +15,7 @@ can be configured using the https://www.elastic.co/guide/en/elasticsearch/refere ** These indices have many conflicting field mappings ** Many of those fields are included in the request These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). +In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. + To work around this issue, you have a number of options: ** Downgrade to an earlier version @@ -23,6 +24,21 @@ To work around this issue, you have a number of options: <> ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. +* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field +but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds: +** If the affected index already contains documents with a dense vector field, force merge the index to a single segment. +** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field +and then force merge to a single segment. +** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields. + +* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or +partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. +There is no workaround and the only option to is to upgrade to 8.15.2 when released. ++ +If you use synthetic source then you may be affected by this bug if the following is true: +** If you have more fields then the `index.mapping.total_fields.limit` setting allows. +** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. + [[bug-8.15.1]] [float] === Bug fixes diff --git a/docs/reference/rest-api/security/get-service-accounts.asciidoc b/docs/reference/rest-api/security/get-service-accounts.asciidoc index 3a14278fb4cfb..74f98f2602e34 100644 --- a/docs/reference/rest-api/security/get-service-accounts.asciidoc +++ b/docs/reference/rest-api/security/get-service-accounts.asciidoc @@ -250,7 +250,8 @@ GET /_security/service/elastic/fleet-server "monitor", "create_index", "auto_configure", - "maintenance" + "maintenance", + "view_index_metadata" ], "allow_restricted_indices": false }, @@ -265,7 +266,8 @@ GET /_security/service/elastic/fleet-server "monitor", "create_index", "auto_configure", - "maintenance" + "maintenance", + "view_index_metadata" ], "allow_restricted_indices": false } diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index 709d17091164c..de9a35e0d29b8 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -36,7 +36,11 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1> { "service": "elser", <2> "service_settings": { - "num_allocations": 1, + "adaptive_allocations": { <3> + "enabled": true, + "min_number_of_allocations": 3, + "max_number_of_allocations": 10 + }, "num_threads": 1 } } @@ -46,6 +50,8 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1> be used and ELSER creates sparse vectors. The `inference_id` is `my-elser-endpoint`. <2> The `elser` service is used in this example. +<3> This setting enables and configures {ml-docs}/ml-nlp-elser.html#elser-adaptive-allocations[adaptive allocations]. +Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process. [NOTE] ==== diff --git a/libs/entitlement-runtime/README.md b/libs/entitlement-runtime/README.md new file mode 100644 index 0000000000000..49cbc873c9de5 --- /dev/null +++ b/libs/entitlement-runtime/README.md @@ -0,0 +1,14 @@ +### Entitlement runtime + +This module implements mechanisms to grant and check permissions under the _entitlements_ system. + +The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal. +The `entitlement-agent` tool instruments sensitive class library methods with calls to this module, in order to enforce the controls. + +This module is responsible for: +- Defining which class library methods are sensitive +- Defining what permissions should be checked for each sensitive method +- Implementing the permission checks +- Offering a "grant" API to grant permissions + +It is not responsible for anything to do with bytecode instrumentation; that responsibility lies with `entitlement-agent`. diff --git a/libs/plugin-classloader/build.gradle b/libs/entitlement-runtime/build.gradle similarity index 58% rename from libs/plugin-classloader/build.gradle rename to libs/entitlement-runtime/build.gradle index f54bec211286a..a552dd7d5ba47 100644 --- a/libs/plugin-classloader/build.gradle +++ b/libs/entitlement-runtime/build.gradle @@ -6,13 +6,19 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ - -// This is only required because :server needs this at runtime. -// We'll be removing this in 8.0 so for now just publish the JAR to make dependency resolution work. +apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' -tasks.named("test").configure { enabled = false } +dependencies { + compileOnly project(':libs:elasticsearch-core') + + testImplementation project(":test:framework") +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} -// test depend on ES core... -tasks.named('forbiddenApisMain').configure { enabled = false} -tasks.named("jarHell").configure { enabled = false } +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/entitlement-runtime/src/main/java/module-info.java b/libs/entitlement-runtime/src/main/java/module-info.java new file mode 100644 index 0000000000000..13849f0658d72 --- /dev/null +++ b/libs/entitlement-runtime/src/main/java/module-info.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.entitlement.runtime { + requires org.elasticsearch.base; + + exports org.elasticsearch.entitlement.runtime.api to org.elasticsearch.entitlement.agent; +} diff --git a/libs/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/api/EntitlementChecks.java b/libs/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/api/EntitlementChecks.java new file mode 100644 index 0000000000000..c06e1e5b1f858 --- /dev/null +++ b/libs/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/api/EntitlementChecks.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.api; + +public class EntitlementChecks { + static boolean isAgentBooted = false; + + public static void setAgentBooted() { + isAgentBooted = true; + } + + public static boolean isAgentBooted() { + return isAgentBooted; + } +} diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index d39cf1cc839f4..1b3aac13b3608 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -45,3 +45,7 @@ dependencies { compileOnly(project(':modules:lang-painless:spi')) clusterModules(project(':modules:lang-painless')) } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed") +}) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml deleted file mode 100644 index dc349171e3a27..0000000000000 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -setup: - - requires: - cluster_features: ["gte_v7.7.0"] - reason: "Start of the week Monday was enabled in a backport to 7.7 PR#50916" - - - do: - indices.create: - index: test - body: - mappings: - properties: - date: - type: date - - - do: - index: - index: test - id: "1" - body: { "date": "2009-11-15T14:12:12" } - - - do: - indices.refresh: - index: [test] - ---- -# The inserted document has a field date=2009-11-15T14:12:12 which is Sunday. -# When aggregating per day of the week this should be considered as last day of the week (7) -# and this value should be used in 'key_as_string' -"Date aggregartion per day of week": - - do: - search: - rest_total_hits_as_int: true - index: test - body: - aggregations: - test: - "date_histogram": { - "field": "date", - "calendar_interval": "day", - "format": "e", - "offset": 0 - } - - - match: {hits.total: 1} - - length: { aggregations.test.buckets: 1 } - - match: { aggregations.test.buckets.0.key_as_string: "7" } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index d20231a6d6cf2..56f387c016261 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -1230,3 +1230,83 @@ non string dimension fields: - match: { .$idx0name.mappings.properties.attributes.properties.double.time_series_dimension: true } - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' } - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.time_series_dimension: true } + +--- +multi value dimensions: + - requires: + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions + + - do: + allowed_warnings: + - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + priority: 1 + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes": { "dim1": ["a" , "b"], "dim2": [1, 2] } }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["b" , "a"], "dim2": [1, 2] } }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["c" , "b"], "dim2": [1, 2] } }' + - is_false: errors + + - do: + search: + index: k9s + body: + size: 0 + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.tsids.buckets: 3 } # only the order of the dim1 attribute is different, yet we expect to have two distinct time series + + - do: + search: + index: k9s + body: + size: 0 + aggs: + dims: + terms: + field: dim1 + order: + _key: asc + + - length: { aggregations.dims.buckets: 3 } + - match: { aggregations.dims.buckets.0.key: a } + - match: { aggregations.dims.buckets.0.doc_count: 2 } + - match: { aggregations.dims.buckets.1.key: b } + - match: { aggregations.dims.buckets.1.doc_count: 3 } + - match: { aggregations.dims.buckets.2.key: c } + - match: { aggregations.dims.buckets.2.doc_count: 1 } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 41e3fad24b681..98dacce01fba4 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -50,3 +50,7 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.commons.logging.LogFactory', ) } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("ingest/30_date_processor/Test week based date parsing", "week-date behaviour has changed") +}) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 600a56a4fa8e2..554e074bcc7e4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -75,17 +75,17 @@ public void testParseJavaDefaultYear() { public void testParseWeekBasedYearAndWeek() { String format = "YYYY-ww"; ZoneId timezone = DateUtils.of("Europe/Amsterdam"); - Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); ZonedDateTime dateTime = javaFunction.apply("2020-33"); - assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 8, 10, 0, 0, 0, 0, timezone))); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 8, 9, 0, 0, 0, 0, timezone))); } public void testParseWeekBasedYear() { String format = "YYYY"; ZoneId timezone = DateUtils.of("Europe/Amsterdam"); - Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); ZonedDateTime dateTime = javaFunction.apply("2019"); - assertThat(dateTime, equalTo(ZonedDateTime.of(2018, 12, 31, 0, 0, 0, 0, timezone))); + assertThat(dateTime, equalTo(ZonedDateTime.of(2018, 12, 30, 0, 0, 0, 0, timezone))); } public void testParseWeekBasedWithLocale() { diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml index 78fb895ea791c..fa095a45891e1 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -246,7 +246,7 @@ teardown: } - length: { docs: 1 } - match: { docs.0.doc._source.date_source_field: "2020-33" } - - match: { docs.0.doc._source.date_target_field: "2020-08-10T00:00:00.000Z" } + - match: { docs.0.doc._source.date_target_field: "2020-08-09T00:00:00.000Z" } - length: { docs.0.doc._ingest: 1 } - is_true: docs.0.doc._ingest.timestamp @@ -262,7 +262,7 @@ teardown: index: test id: "1" - match: { _source.date_source_field: "2020-33" } - - match: { _source.date_target_field: "2020-08-10T00:00:00.000Z" } + - match: { _source.date_target_field: "2020-08-09T00:00:00.000Z" } --- "Test week based date parsing with locale": diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java index 3c9e684ef4279..021ce09e0ed8e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java @@ -46,7 +46,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception assert msg instanceof HttpObject; if (msg instanceof HttpRequest request) { var preReq = HttpHeadersAuthenticatorUtils.asHttpPreRequest(request); - aggregating = decider.test(preReq) && IGNORE_TEST.test(preReq); + aggregating = (decider.test(preReq) && IGNORE_TEST.test(preReq)) || request.decoderResult().isFailure(); } if (aggregating || msg instanceof FullHttpRequest) { super.channelRead(ctx, msg); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java index b04da46a2d7d7..a1aa211814520 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -17,6 +17,7 @@ import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.QueryStringDecoder; import io.netty.handler.codec.http.cookie.Cookie; import io.netty.handler.codec.http.cookie.ServerCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; @@ -48,6 +49,7 @@ public class Netty4HttpRequest implements HttpRequest { private final Exception inboundException; private final boolean pooled; private final int sequence; + private final QueryStringDecoder queryStringDecoder; Netty4HttpRequest(int sequence, io.netty.handler.codec.http.HttpRequest request, Netty4HttpRequestBodyStream contentStream) { this( @@ -94,6 +96,7 @@ private Netty4HttpRequest( this.pooled = pooled; this.released = released; this.inboundException = inboundException; + this.queryStringDecoder = new QueryStringDecoder(request.uri()); } @Override @@ -106,6 +109,11 @@ public String uri() { return request.uri(); } + @Override + public String rawPath() { + return queryStringDecoder.rawPath(); + } + @Override public HttpBody body() { assert released.get() == false; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index c6e7fa3517771..5ed3d81392951 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -375,9 +375,8 @@ protected HttpMessage createMessage(String[] initialLine) throws Exception { final HttpObjectAggregator aggregator = new Netty4HttpAggregator( handlingSettings.maxContentLength(), httpPreRequest -> enabled.get() == false - || (httpPreRequest.uri().contains("_bulk") == false - || httpPreRequest.uri().contains("_bulk_update") - || httpPreRequest.uri().contains("/_xpack/monitoring/_bulk")) + || ((httpPreRequest.rawPath().endsWith("/_bulk") == false) + || httpPreRequest.rawPath().startsWith("/_xpack/monitoring/_bulk")) ); aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); ch.pipeline() diff --git a/muted-tests.yml b/muted-tests.yml index a2dcfdda9a2c1..e9b359467282e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -215,9 +215,6 @@ tests: - class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT method: test {case-functions.testSelectInsertWithLcaseAndLengthWithOrderBy} issue: https://github.com/elastic/elasticsearch/issues/112642 -- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT - method: testHistogramAggregation - issue: https://github.com/elastic/elasticsearch/issues/113109 - class: org.elasticsearch.action.admin.cluster.node.stats.NodeStatsTests method: testChunking issue: https://github.com/elastic/elasticsearch/issues/113139 @@ -235,18 +232,6 @@ tests: - class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests method: testErrorMidStream issue: https://github.com/elastic/elasticsearch/issues/113179 -- class: org.elasticsearch.logsdb.datageneration.DataGeneratorTests - method: testDataGeneratorProducesValidMappingAndDocument - issue: https://github.com/elastic/elasticsearch/issues/112966 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with Italian locale} - issue: https://github.com/elastic/elasticsearch/issues/113198 -- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT - method: test {date.DateFormatLocale SYNC} - issue: https://github.com/elastic/elasticsearch/issues/113199 -- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT - method: test {date.DateFormatLocale ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/113200 - class: org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests method: testHasPrivilegesOtherThanIndex issue: https://github.com/elastic/elasticsearch/issues/113202 @@ -271,15 +256,86 @@ tests: - class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests method: "testFold {TestCase= #2}" issue: https://github.com/elastic/elasticsearch/issues/113225 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with default locale} - issue: https://github.com/elastic/elasticsearch/issues/113226 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates} - issue: https://github.com/elastic/elasticsearch/issues/113227 - class: org.elasticsearch.index.mapper.DoubleRangeFieldMapperTests method: testSyntheticSourceKeepAll issue: https://github.com/elastic/elasticsearch/issues/113234 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testGetMappings + issue: https://github.com/elastic/elasticsearch/issues/113260 +- class: org.elasticsearch.xpack.security.authz.SecurityScrollTests + method: testSearchAndClearScroll + issue: https://github.com/elastic/elasticsearch/issues/113285 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date.EvalDateFormatString SYNC} + issue: https://github.com/elastic/elasticsearch/issues/113293 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {date.EvalDateFormat} + issue: https://github.com/elastic/elasticsearch/issues/113294 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {date.DocsDateFormat} + issue: https://github.com/elastic/elasticsearch/issues/113295 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {stats.DocsStatsGroupByMultipleValues} + issue: https://github.com/elastic/elasticsearch/issues/113296 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + issue: https://github.com/elastic/elasticsearch/issues/113298 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testGetIndex + issue: https://github.com/elastic/elasticsearch/issues/113311 +- class: org.elasticsearch.packaging.test.WindowsServiceTests + method: test81JavaOptsInJvmOptions + issue: https://github.com/elastic/elasticsearch/issues/113313 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=esql/50_index_patterns/disjoint_mappings} + issue: https://github.com/elastic/elasticsearch/issues/113315 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=wildcard/10_wildcard_basic/Query_string wildcard query} + issue: https://github.com/elastic/elasticsearch/issues/113316 +- class: org.elasticsearch.index.mapper.LongRangeFieldMapperTests + method: testSyntheticSourceKeepAll + issue: https://github.com/elastic/elasticsearch/issues/113324 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item} + issue: https://github.com/elastic/elasticsearch/issues/113325 +- class: org.elasticsearch.index.mapper.IntegerRangeFieldMapperTests + method: testSyntheticSourceKeepArrays + issue: https://github.com/elastic/elasticsearch/issues/113326 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=transform/transforms_force_delete/Test force deleting a running transform} + issue: https://github.com/elastic/elasticsearch/issues/113327 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testValidateQuery + issue: https://github.com/elastic/elasticsearch/issues/113328 +- class: org.elasticsearch.index.mapper.LongRangeFieldMapperTests + method: testSyntheticSourceKeepArrays + issue: https://github.com/elastic/elasticsearch/issues/113335 +- class: org.elasticsearch.xpack.security.support.SecurityIndexManagerIntegTests + method: testOnIndexAvailableForSearchIndexAlreadyAvailable + issue: https://github.com/elastic/elasticsearch/issues/113336 +- class: org.elasticsearch.xpack.security.authz.SecurityScrollTests + method: testScrollIsPerUser + issue: https://github.com/elastic/elasticsearch/issues/113338 +- class: org.elasticsearch.index.mapper.IntegerRangeFieldMapperTests + method: testSyntheticSourceKeepAll + issue: https://github.com/elastic/elasticsearch/issues/113339 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=analytics/top_metrics/sort by scaled float field} + issue: https://github.com/elastic/elasticsearch/issues/113340 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testFieldMappings + issue: https://github.com/elastic/elasticsearch/issues/113341 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/ccr/apis/follow/post-resume-follow/line_84} + issue: https://github.com/elastic/elasticsearch/issues/113343 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testSearchAndMSearch + issue: https://github.com/elastic/elasticsearch/issues/113345 +- class: org.elasticsearch.action.bulk.IncrementalBulkIT + method: testBulkLevelBulkFailureAfterFirstIncrementalRequest + issue: https://github.com/elastic/elasticsearch/issues/113365 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testDeleteJob_TimingStatsDocumentIsDeleted + issue: https://github.com/elastic/elasticsearch/issues/113370 # Examples: # diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java index a7fa63709d580..edb29a8f4c98e 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java @@ -33,6 +33,7 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory { private static final String NBEST_COST = "nbest_cost"; private static final String NBEST_EXAMPLES = "nbest_examples"; private static final String DISCARD_COMPOUND_TOKEN = "discard_compound_token"; + private static final String LENIENT = "lenient"; private final UserDictionary userDictionary; private final Mode mode; @@ -58,7 +59,15 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting "It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" + " with [" + USER_DICT_RULES_OPTION + "]" ); } - List ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, false, true); + List ruleList = Analysis.getWordList( + env, + settings, + USER_DICT_PATH_OPTION, + USER_DICT_RULES_OPTION, + LENIENT, + false, // typically don't want to remove comments as deduplication will provide better feedback + true + ); if (ruleList == null || ruleList.isEmpty()) { return null; } @@ -66,6 +75,7 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting for (String line : ruleList) { sb.append(line).append(System.lineSeparator()); } + try (Reader rulesReader = new StringReader(sb.toString())) { return UserDictionary.open(rulesReader); } catch (IOException e) { diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java index 1229b4f348911..f26213d86c5a9 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java @@ -445,7 +445,26 @@ public void testKuromojiAnalyzerDuplicateUserDictRule() throws Exception { ) .build(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings)); - assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [3]")); + assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [4]")); + } + + public void testKuromojiAnalyzerDuplicateUserDictRuleDeduplication() throws Exception { + Settings settings = Settings.builder() + .put("index.analysis.analyzer.my_analyzer.type", "kuromoji") + .put("index.analysis.analyzer.my_analyzer.lenient", "true") + .putList( + "index.analysis.analyzer.my_analyzer.user_dictionary_rules", + "c++,c++,w,w", + "#comment", + "制限スピード,制限スピード,セイゲンスピード,テスト名詞", + "制限スピード,制限スピード,セイゲンスピード,テスト名詞" + ) + .build(); + TestAnalysis analysis = createTestAnalysis(settings); + Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer"); + try (TokenStream stream = analyzer.tokenStream("", "制限スピード")) { + assertTokenStreamContents(stream, new String[] { "制限スピード" }); + } } public void testDiscardCompoundToken() throws Exception { diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java index 8bc53fa69c9a7..ed8458bc94043 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java @@ -31,6 +31,7 @@ public class NoriTokenizerFactory extends AbstractTokenizerFactory { private static final String USER_DICT_PATH_OPTION = "user_dictionary"; private static final String USER_DICT_RULES_OPTION = "user_dictionary_rules"; + private static final String LENIENT = "lenient"; private final UserDictionary userDictionary; private final KoreanTokenizer.DecompoundMode decompoundMode; @@ -54,7 +55,8 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, - true, + LENIENT, + false, // typically don't want to remove comments as deduplication will provide better feedback isSupportDuplicateCheck(indexSettings) ); if (ruleList == null || ruleList.isEmpty()) { diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java index e1123f167da99..1709d02263eea 100644 --- a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java +++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java @@ -127,7 +127,7 @@ public void testNoriAnalyzerDuplicateUserDictRule() throws Exception { .build(); final IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings)); - assertThat(exc.getMessage(), containsString("[세종] in user dictionary at line [3]")); + assertThat(exc.getMessage(), containsString("[세종] in user dictionary at line [4]")); } public void testNoriAnalyzerDuplicateUserDictRuleWithLegacyVersion() throws IOException { @@ -144,6 +144,20 @@ public void testNoriAnalyzerDuplicateUserDictRuleWithLegacyVersion() throws IOEx } } + public void testNoriAnalyzerDuplicateUserDictRuleDeduplication() throws Exception { + Settings settings = Settings.builder() + .put("index.analysis.analyzer.my_analyzer.type", "nori") + .put("index.analysis.analyzer.my_analyzer.lenient", "true") + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.NORI_DUPLICATES) + .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++", "C쁠쁠", "세종", "세종", "세종시 세종 시") + .build(); + TestAnalysis analysis = createTestAnalysis(settings); + Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer"); + try (TokenStream stream = analyzer.tokenStream("", "세종시")) { + assertTokenStreamContents(stream, new String[] { "세종", "시" }); + } + } + public void testNoriTokenizer() throws Exception { Settings settings = Settings.builder() .put("index.analysis.tokenizer.my_tokenizer.type", "nori_tokenizer") diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java index 08026e0435f33..2b24e53874e51 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java @@ -29,6 +29,12 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) public class IncrementalBulkRestIT extends HttpSmokeTestCase { + public void testBulkUriMatchingDoesNotMatchBulkCapabilitiesApi() throws IOException { + Request request = new Request("GET", "/_capabilities?method=GET&path=%2F_bulk&capabilities=failure_store_status&pretty"); + Response response = getRestClient().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + public void testBulkMissingBody() throws IOException { Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"); request.setJsonEntity(""); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java index c652a9992d8fb..322c055de140b 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -102,22 +101,9 @@ private void runTest(String actionName, String endpoint) throws Exception { private void updateClusterState(Function transformationFn) { final TimeValue timeout = TimeValue.timeValueSeconds(10); - - final AckedRequest ackedRequest = new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return timeout; - } - - @Override - public TimeValue masterNodeTimeout() { - return timeout; - } - }; - PlainActionFuture future = new PlainActionFuture<>(); internalCluster().getAnyMasterNodeInstance(ClusterService.class) - .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(ackedRequest, future) { + .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(timeout, timeout, future) { @Override public ClusterState execute(ClusterState currentState) throws Exception { return transformationFn.apply(currentState); diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 015c9c4b812c6..eb3d66ece50d6 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -53,3 +53,7 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention' } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") +}) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 8a27eb3efd219..b5a9146bc54a6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1269,8 +1269,8 @@ synthetic_source with copy_to and ignored values: refresh: true body: name: "B" - k: ["5", "6"] - long: ["7", "8"] + k: ["55", "66"] + long: ["77", "88"] - do: search: @@ -1289,10 +1289,9 @@ synthetic_source with copy_to and ignored values: - match: hits.hits.1._source: name: "B" - k: ["5", "6"] - long: ["7", "8"] - - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } - + k: ["55", "66"] + long: ["77", "88"] + - match: { hits.hits.1.fields.copy: ["55", "66", "77", "88"] } --- synthetic_source with copy_to field having values in source: @@ -1553,3 +1552,420 @@ synthetic_source with copy_to and invalid values for copy: - match: { error.type: "document_parsing_exception" } - contains: { error.reason: "Copy-to currently only works for value-type fields" } + +--- +synthetic_source with copy_to pointing inside object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + my_values: + properties: + k: + type: keyword + ignore_above: 1 + copy_to: c.copy + long: + type: long + copy_to: c.copy + c: + properties: + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + my_values: + k: "hello" + long: 100 + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy" ] + + - match: + hits.hits.0._source: + name: "A" + my_values: + k: "hello" + long: 100 + - match: + hits.hits.0.fields: + c.copy: [ "100", "hello" ] + + - match: + hits.hits.1._source: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + - match: + hits.hits.1.fields: + c.copy: ["55", "66", "77", "88"] + + - match: + hits.hits.2._source: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy: [ "100", "hello", "zap" ] + +--- +synthetic_source with copy_to pointing to ambiguous field: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects false: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: false + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects auto: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: auto + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing at dynamic field: + - requires: + test_runner_features: contains + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + k: + type: keyword + copy_to: c.copy + c: + properties: + f: + type: float + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + name: "B" + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + name: "C" + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + +--- +synthetic_source with copy_to pointing inside dynamic object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + k: + type: keyword + copy_to: c.copy + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + name: "B" + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + name: "C" + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 7c345b7d4d3ac..975113953c995 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -1,6 +1,10 @@ --- "Test Index and Search locale dependent mappings / dates": + - requires: + test_runner_features: ["allowed_warnings"] - do: + allowed_warnings: + - "Date format [E, d MMM yyyy HH:mm:ss Z] contains textual field specifiers that could change in JDK 23" indices.create: index: test_index body: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 6eb7a8dcad7aa..616afd3cf67ad 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -119,11 +119,11 @@ missing dimension on routing path field: type: keyword --- -multi-value routing path field: +multi-value routing path field succeeds: - requires: test_runner_features: close_to - cluster_features: ["gte_v8.13.0"] - reason: _tsid hashing introduced in 8.13 + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions - do: indices.create: @@ -172,12 +172,7 @@ multi-value routing path field: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "unmapped_field": 40, "tag": [ "one", "three" ] }' - - is_true: errors - - - match: {items.1.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.3.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.4.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.7.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } + - is_false: errors - do: search: @@ -195,13 +190,21 @@ multi-value routing path field: avg: field: voltage - - match: {hits.total.value: 4} - - length: {aggregations.tsids.buckets: 2} + - match: {hits.total.value: 8} + - length: {aggregations.tsids.buckets: 4} - - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSOYoRw_LI4DD7DFEGEJ3NR3eQkMY" } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSddqA4WYKglGPR_C0cJe8QGaiC2c" } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} + + - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: {aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} + + - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: {aggregations.tsids.buckets.3.doc_count: 2 } + - close_to: {aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/server/build.gradle b/server/build.gradle index 5492ca00e2d3b..5c12d47da8102 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -40,8 +40,6 @@ dependencies { api project(":libs:elasticsearch-tdigest") implementation project(":libs:elasticsearch-simdvec") - implementation project(':libs:elasticsearch-plugin-classloader') - // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" api "org.apache.lucene:lucene-analysis-common:${versions.lucene}" diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index ac1c44de6dcf6..2a275cf563d86 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -255,9 +255,9 @@ private Set getShardIds(final String nodeId, final String indexName) { /** * Index documents until all the shards are at least WATERMARK_BYTES in size, and return the one with the smallest size */ - private ShardSizes createReasonableSizedShards(final String indexName) throws InterruptedException { + private ShardSizes createReasonableSizedShards(final String indexName) { while (true) { - indexRandom(true, indexName, scaledRandomIntBetween(100, 10000)); + indexRandom(false, indexName, scaledRandomIntBetween(100, 10000)); forceMerge(); refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 61cf49ff6ca4e..9f1a094c80623 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; @@ -86,6 +87,7 @@ import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.recovery.RecoveryStats; @@ -137,8 +139,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.LongSupplier; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import static java.util.Collections.singletonMap; import static java.util.stream.Collectors.toList; @@ -146,7 +150,10 @@ import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED; import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; +import static org.elasticsearch.index.MergePolicyConfig.INDEX_MERGE_ENABLED; import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; +import static org.elasticsearch.indices.IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING; +import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -158,6 +165,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -1957,6 +1965,77 @@ public void accept(long globalCheckpoint, Exception e) { recoveryCompleteListener.onResponse(null); } + public void testPostRecoveryMerge() throws Exception { + internalCluster().startMasterOnlyNode(); + final var dataNode = internalCluster().startDataOnlyNode(); + final var indexName = randomIdentifier(); + createIndex(indexName, indexSettings(1, 0).put(INDEX_MERGE_ENABLED, false).build()); + + final var initialSegmentCount = 20; + for (int i = 0; i < initialSegmentCount; i++) { + indexDoc(indexName, Integer.toString(i), "f", randomAlphaOfLength(10)); + refresh(indexName); // force a one-doc segment + } + flush(indexName); // commit all the one-doc segments + + final LongSupplier searchableSegmentCountSupplier = () -> indicesAdmin().prepareSegments(indexName) + .get(SAFE_AWAIT_TIMEOUT) + .getIndices() + .get(indexName) + .getShards() + .get(0) + .shards()[0].getSegments() + .stream() + .filter(Segment::isSearch) + .count(); + + assertEquals(initialSegmentCount, searchableSegmentCountSupplier.getAsLong()); + + // force a recovery by restarting the node, re-enabling merges while the node is down, but configure the node not to be in the hot + // or content tiers so that it does not do any post-recovery merge + internalCluster().restartNode(dataNode, new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) { + final var request = new UpdateSettingsRequest(Settings.builder().putNull(INDEX_MERGE_ENABLED).build(), indexName); + request.reopen(true); + safeGet(indicesAdmin().updateSettings(request)); + return Settings.builder() + .putList(NODE_ROLES_SETTING.getKey(), randomNonEmptySubsetOf(List.of("data_warm", "data_cold"))) + .build(); + } + }); + + ensureGreen(indexName); + final var mergeStats = indicesAdmin().prepareStats(indexName).clear().setMerge(true).get().getIndex(indexName).getShards()[0] + .getStats() + .getMerge(); + assertEquals(0, mergeStats.getCurrent()); + assertEquals(0, mergeStats.getTotal()); + assertEquals(initialSegmentCount, searchableSegmentCountSupplier.getAsLong()); + + // force a recovery by restarting the node again, but this time putting it into the hot or content tiers to enable post-recovery + // merges + internalCluster().restartNode(dataNode, new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) { + return Settings.builder() + .putList( + NODE_ROLES_SETTING.getKey(), + Stream.concat( + Stream.of(randomFrom("data", "data_content", "data_hot")), + Stream.of("data", "data_content", "data_hot", "data_warm", "data_cold").filter(p -> randomBoolean()) + ).distinct().toList() + ) + // set the inactive time to zero so that we flush immediately after every merge, rather than having the test wait 5min + .put(SHARD_INACTIVE_TIME_SETTING.getKey(), TimeValue.ZERO) + .build(); + } + }); + + ensureGreen(indexName); + assertBusy(() -> assertThat(searchableSegmentCountSupplier.getAsLong(), lessThan((long) initialSegmentCount))); + } + private void assertGlobalCheckpointIsStableAndSyncedInAllNodes(String indexName, List nodes, int shard) throws Exception { assertThat(nodes, is(not(empty()))); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java index 61a1168b128b9..de62c0152817a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java @@ -18,7 +18,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; @@ -33,9 +32,7 @@ import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.core.PathUtils; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.discovery.AbstractDisruptionTestCase; -import org.elasticsearch.index.Index; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoryConflictException; @@ -59,6 +56,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -2214,12 +2212,17 @@ public void testDeleteIndexWithOutOfOrderFinalization() { .anyMatch(e -> e.snapshot().getSnapshotId().getName().equals("snapshot-with-index-1") && e.state().completed()) ) // execute the index deletion _directly on the master_ so it happens before the snapshot finalization executes - .andThen(l -> masterDeleteIndexService.deleteIndices(new DeleteIndexClusterStateUpdateRequest(l.map(r -> { - assertTrue(r.isAcknowledged()); - return null; - })).indices(new Index[] { internalCluster().clusterService().state().metadata().index(indexToDelete).getIndex() }) - .ackTimeout(TimeValue.timeValueSeconds(10)) - .masterNodeTimeout(TimeValue.timeValueSeconds(10)))) + .andThen( + l -> masterDeleteIndexService.deleteIndices( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + Set.of(internalCluster().clusterService().state().metadata().index(indexToDelete).getIndex()), + l.map(r -> { + assertTrue(r.isAcknowledged()); + return null; + }) + ) + ) // ultimately create the index again so that taking a full snapshot will pick up any missing shard gen blob, and deleting that // full snapshot will clean up all dangling shard-level blobs .andThen((l, ignored) -> prepareCreate(indexToDelete, indexSettingsNoReplicas(1)).execute(l.map(r -> { diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 696624a4a8f27..f60a595d94c11 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -25,7 +25,6 @@ requires org.elasticsearch.nativeaccess; requires org.elasticsearch.geo; requires org.elasticsearch.lz4; - requires org.elasticsearch.pluginclassloader; requires org.elasticsearch.securesm; requires org.elasticsearch.xcontent; requires org.elasticsearch.logging; @@ -405,7 +404,6 @@ exports org.elasticsearch.telemetry; exports org.elasticsearch.telemetry.metric; - provides java.util.spi.CalendarDataProvider with org.elasticsearch.common.time.IsoCalendarDataProvider; provides org.elasticsearch.xcontent.ErrorOnUnknown with org.elasticsearch.common.xcontent.SuggestingErrorOnUnknown; provides org.elasticsearch.xcontent.XContentBuilderExtension with org.elasticsearch.common.xcontent.XContentElasticsearchExtension; provides org.elasticsearch.cli.CliToolProvider diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 2a628466ea396..a4fda469da3a2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -16,7 +16,9 @@ import org.elasticsearch.action.admin.cluster.allocation.TransportGetAllocationStatsAction; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; @@ -47,7 +49,7 @@ public class TransportNodesStatsAction extends TransportNodesAction< NodesStatsResponse, TransportNodesStatsAction.NodeStatsRequest, NodeStats, - Void> { + SubscribableListener> { public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/stats"); @@ -77,37 +79,55 @@ public TransportNodesStatsAction( @Override protected NodesStatsResponse newResponse(NodesStatsRequest request, List responses, List failures) { - return new NodesStatsResponse(clusterService.getClusterName(), responses, failures); + assert false; + throw new UnsupportedOperationException("use newResponseAsync instead"); + } + + @Override + protected SubscribableListener createActionContext(Task task, NodesStatsRequest request) { + return SubscribableListener.newForked(l -> { + var metrics = request.getNodesStatsRequestParameters().requestedMetrics(); + if (metrics.contains(Metric.FS) || metrics.contains(Metric.ALLOCATIONS)) { + new ParentTaskAssigningClient(client, clusterService.localNode(), task).execute( + TransportGetAllocationStatsAction.TYPE, + new TransportGetAllocationStatsAction.Request( + Objects.requireNonNullElse(request.timeout(), RestUtils.REST_MASTER_TIMEOUT_DEFAULT), + new TaskId(clusterService.localNode().getId(), task.getId()), + metrics + ), + l + ); + } else { + l.onResponse(null); + } + }); } @Override protected void newResponseAsync( Task task, NodesStatsRequest request, - Void actionContext, + SubscribableListener actionContext, List responses, List failures, ActionListener listener ) { - var metrics = request.getNodesStatsRequestParameters().requestedMetrics(); - if (metrics.contains(Metric.FS) || metrics.contains(Metric.ALLOCATIONS)) { - client.execute( - TransportGetAllocationStatsAction.TYPE, - new TransportGetAllocationStatsAction.Request( - Objects.requireNonNullElse(request.timeout(), RestUtils.REST_MASTER_TIMEOUT_DEFAULT), - new TaskId(clusterService.localNode().getId(), task.getId()), - metrics - ), - listener.delegateFailure( - (l, r) -> ActionListener.respondAndRelease( - l, - newResponse(request, merge(responses, r.getNodeAllocationStats(), r.getDiskThresholdSettings()), failures) - ) + actionContext + // merge in the stats from the master, if available + .andThenApply( + getAllocationStatsResponse -> new NodesStatsResponse( + clusterService.getClusterName(), + getAllocationStatsResponse == null + ? responses + : merge( + responses, + getAllocationStatsResponse.getNodeAllocationStats(), + getAllocationStatsResponse.getDiskThresholdSettings() + ), + failures ) - ); - } else { - ActionListener.run(listener, l -> ActionListener.respondAndRelease(l, newResponse(request, responses, failures))); - } + ) + .addListener(listener); } private static List merge( diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java index e98091beb88a5..c6d60f34eddeb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java @@ -9,32 +9,25 @@ package org.elasticsearch.action.admin.indices.alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse.AliasActionResult; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.AliasAction; +import org.elasticsearch.core.TimeValue; import java.util.List; +import java.util.Objects; /** * Cluster state update request that allows to add or remove aliases */ -public class IndicesAliasesClusterStateUpdateRequest extends ClusterStateUpdateRequest { - private final List actions; - - private final List actionResults; - - public IndicesAliasesClusterStateUpdateRequest(List actions, List actionResults) { - this.actions = actions; - this.actionResults = actionResults; - } - - /** - * Returns the alias actions to be performed - */ - public List actions() { - return actions; - } - - public List getActionResults() { - return actionResults; +public record IndicesAliasesClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + List actions, + List actionResults +) { + public IndicesAliasesClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(actions); + Objects.requireNonNull(actionResults); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 129e853ec638c..c73c44e9a23e7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -255,9 +255,11 @@ protected void masterOperation( } request.aliasActions().clear(); IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), unmodifiableList(finalActions), unmodifiableList(actionResults) - ).ackTimeout(request.ackTimeout()).masterNodeTimeout(request.masterNodeTimeout()); + ); indexAliasesService.indicesAliases(updateRequest, listener.delegateResponse((l, e) -> { logger.debug("failed to perform aliases", e); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java deleted file mode 100644 index cc6971a0f584b..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package org.elasticsearch.action.admin.indices.delete; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.ClusterStateAckListener; -import org.elasticsearch.cluster.ClusterStateTaskListener; -import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest; -import org.elasticsearch.cluster.node.DiscoveryNode; - -/** - * Cluster state update request that allows to close one or more indices - */ -public class DeleteIndexClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest - implements - ClusterStateAckListener, - ClusterStateTaskListener { - - private final ActionListener listener; - - public DeleteIndexClusterStateUpdateRequest(ActionListener listener) { - this.listener = listener; - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - - @Override - public boolean mustAck(DiscoveryNode discoveryNode) { - return true; - } - - @Override - public void onAllNodesAcked() { - listener.onResponse(AcknowledgedResponse.TRUE); - } - - @Override - public void onAckFailure(Exception e) { - listener.onResponse(AcknowledgedResponse.FALSE); - } - - @Override - public void onAckTimeout() { - listener.onResponse(AcknowledgedResponse.FALSE); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java index 24c152c1a1947..fbf95699dd03a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -92,11 +92,14 @@ protected void masterOperation( return; } - DeleteIndexClusterStateUpdateRequest deleteRequest = new DeleteIndexClusterStateUpdateRequest(listener.delegateResponse((l, e) -> { - logger.debug(() -> "failed to delete indices [" + concreteIndices + "]", e); - listener.onFailure(e); - })).ackTimeout(request.ackTimeout()).masterNodeTimeout(request.masterNodeTimeout()).indices(concreteIndices.toArray(new Index[0])); - - deleteIndexService.deleteIndices(deleteRequest); + deleteIndexService.deleteIndices( + request.masterNodeTimeout(), + request.ackTimeout(), + concreteIndices, + listener.delegateResponse((l, e) -> { + logger.debug(() -> "failed to delete indices [" + concreteIndices + "]", e); + listener.onFailure(e); + }) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index dcb28b28f2b76..772f36898202b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.core.Nullable; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.AggregatedDfs; @@ -39,13 +40,15 @@ final class FetchSearchPhase extends SearchPhase { private final Logger logger; private final SearchProgressListener progressListener; private final AggregatedDfs aggregatedDfs; + @Nullable + private final SearchPhaseResults resultConsumer; private final SearchPhaseController.ReducedQueryPhase reducedQueryPhase; FetchSearchPhase( SearchPhaseResults resultConsumer, AggregatedDfs aggregatedDfs, SearchPhaseContext context, - SearchPhaseController.ReducedQueryPhase reducedQueryPhase + @Nullable SearchPhaseController.ReducedQueryPhase reducedQueryPhase ) { this( resultConsumer, @@ -64,7 +67,7 @@ final class FetchSearchPhase extends SearchPhase { SearchPhaseResults resultConsumer, AggregatedDfs aggregatedDfs, SearchPhaseContext context, - SearchPhaseController.ReducedQueryPhase reducedQueryPhase, + @Nullable SearchPhaseController.ReducedQueryPhase reducedQueryPhase, BiFunction, SearchPhase> nextPhaseFactory ) { super("fetch"); @@ -85,6 +88,7 @@ final class FetchSearchPhase extends SearchPhase { this.logger = context.getLogger(); this.progressListener = context.getTask().getProgressListener(); this.reducedQueryPhase = reducedQueryPhase; + this.resultConsumer = reducedQueryPhase == null ? resultConsumer : null; } @Override @@ -92,7 +96,7 @@ public void run() { context.execute(new AbstractRunnable() { @Override - protected void doRun() { + protected void doRun() throws Exception { innerRun(); } @@ -103,7 +107,10 @@ public void onFailure(Exception e) { }); } - private void innerRun() { + private void innerRun() throws Exception { + assert this.reducedQueryPhase == null ^ this.resultConsumer == null; + // depending on whether we executed the RankFeaturePhase we may or may not have the reduced query result computed already + final var reducedQueryPhase = this.reducedQueryPhase == null ? resultConsumer.reduce() : this.reducedQueryPhase; final int numShards = context.getNumShards(); // Usually when there is a single shard, we force the search type QUERY_THEN_FETCH. But when there's kNN, we might // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase. @@ -113,7 +120,7 @@ private void innerRun() { if (queryAndFetchOptimization) { assert assertConsistentWithQueryAndFetchOptimization(); // query AND fetch optimization - moveToNextPhase(searchPhaseShardResults); + moveToNextPhase(searchPhaseShardResults, reducedQueryPhase); } else { ScoreDoc[] scoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // no docs to fetch -- sidestep everything and return @@ -121,7 +128,7 @@ private void innerRun() { // we have to release contexts here to free up resources searchPhaseShardResults.asList() .forEach(searchPhaseShardResult -> releaseIrrelevantSearchContext(searchPhaseShardResult, context)); - moveToNextPhase(fetchResults.getAtomicArray()); + moveToNextPhase(fetchResults.getAtomicArray(), reducedQueryPhase); } else { final boolean shouldExplainRank = shouldExplainRankScores(context.getRequest()); final List> rankDocsPerShard = false == shouldExplainRank @@ -134,7 +141,7 @@ private void innerRun() { final CountedCollector counter = new CountedCollector<>( fetchResults, docIdsToLoad.length, // we count down every shard in the result no matter if we got any results or not - () -> moveToNextPhase(fetchResults.getAtomicArray()), + () -> moveToNextPhase(fetchResults.getAtomicArray(), reducedQueryPhase), context ); for (int i = 0; i < docIdsToLoad.length; i++) { @@ -243,7 +250,10 @@ public void onFailure(Exception e) { ); } - private void moveToNextPhase(AtomicArray fetchResultsArr) { + private void moveToNextPhase( + AtomicArray fetchResultsArr, + SearchPhaseController.ReducedQueryPhase reducedQueryPhase + ) { var resp = SearchPhaseController.merge(context.getRequest().scroll() != null, reducedQueryPhase, fetchResultsArr); context.addReleasable(resp::decRef); fetchResults.close(); diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index 49e5c1b6d69e3..81053a70eca9f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -70,6 +70,12 @@ public class RankFeaturePhase extends SearchPhase { @Override public void run() { + RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source()); + if (rankFeaturePhaseRankCoordinatorContext == null) { + moveToNextPhase(queryPhaseResults, null); + return; + } + context.execute(new AbstractRunnable() { @Override protected void doRun() throws Exception { @@ -77,7 +83,7 @@ protected void doRun() throws Exception { // was set up at FetchSearchPhase. // we do the heavy lifting in this inner run method where we reduce aggs etc - innerRun(); + innerRun(rankFeaturePhaseRankCoordinatorContext); } @Override @@ -87,51 +93,39 @@ public void onFailure(Exception e) { }); } - void innerRun() throws Exception { + void innerRun(RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext) throws Exception { // if the RankBuilder specifies a QueryPhaseCoordinatorContext, it will be called as part of the reduce call // to operate on the first `rank_window_size * num_shards` results and merge them appropriately. SearchPhaseController.ReducedQueryPhase reducedQueryPhase = queryPhaseResults.reduce(); - RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source()); - if (rankFeaturePhaseRankCoordinatorContext != null) { - ScoreDoc[] queryScoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // rank_window_size - final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(context.getNumShards(), queryScoreDocs); - final CountedCollector rankRequestCounter = new CountedCollector<>( - rankPhaseResults, - context.getNumShards(), - () -> onPhaseDone(rankFeaturePhaseRankCoordinatorContext, reducedQueryPhase), - context - ); + ScoreDoc[] queryScoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // rank_window_size + final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(context.getNumShards(), queryScoreDocs); + final CountedCollector rankRequestCounter = new CountedCollector<>( + rankPhaseResults, + context.getNumShards(), + () -> onPhaseDone(rankFeaturePhaseRankCoordinatorContext, reducedQueryPhase), + context + ); - // we send out a request to each shard in order to fetch the needed feature info - for (int i = 0; i < docIdsToLoad.length; i++) { - List entry = docIdsToLoad[i]; - SearchPhaseResult queryResult = queryPhaseResults.getAtomicArray().get(i); - if (entry == null || entry.isEmpty()) { - if (queryResult != null) { - releaseIrrelevantSearchContext(queryResult, context); - progressListener.notifyRankFeatureResult(i); - } - rankRequestCounter.countDown(); - } else { - executeRankFeatureShardPhase(queryResult, rankRequestCounter, entry); + // we send out a request to each shard in order to fetch the needed feature info + for (int i = 0; i < docIdsToLoad.length; i++) { + List entry = docIdsToLoad[i]; + SearchPhaseResult queryResult = queryPhaseResults.getAtomicArray().get(i); + if (entry == null || entry.isEmpty()) { + if (queryResult != null) { + releaseIrrelevantSearchContext(queryResult, context); + progressListener.notifyRankFeatureResult(i); } + rankRequestCounter.countDown(); + } else { + executeRankFeatureShardPhase(queryResult, rankRequestCounter, entry); } - } else { - moveToNextPhase(queryPhaseResults, reducedQueryPhase); } } private RankFeaturePhaseRankCoordinatorContext coordinatorContext(SearchSourceBuilder source) { return source == null || source.rankBuilder() == null ? null - : context.getRequest() - .source() - .rankBuilder() - .buildRankFeaturePhaseCoordinatorContext( - context.getRequest().source().size(), - context.getRequest().source().from(), - client - ); + : source.rankBuilder().buildRankFeaturePhaseCoordinatorContext(source.size(), source.from(), client); } private void executeRankFeatureShardPhase( diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java index a69c4470c2add..11872cd85d9ba 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.support.master; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; @@ -23,9 +22,7 @@ * Abstract base class for action requests that track acknowledgements of cluster state updates: such a request is acknowledged only once * the cluster state update is committed and all relevant nodes have applied it and acknowledged its application to the elected master.. */ -public abstract class AcknowledgedRequest> extends MasterNodeRequest - implements - AckedRequest { +public abstract class AcknowledgedRequest> extends MasterNodeRequest { public static final TimeValue DEFAULT_ACK_TIMEOUT = timeValueSeconds(30); @@ -74,7 +71,6 @@ public final Request ackTimeout(TimeValue ackTimeout) { /** * @return the current ack timeout as a {@link TimeValue} */ - @Override public final TimeValue ackTimeout() { return ackTimeout; } diff --git a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java index 34d7b6e913dec..de42591c15d27 100644 --- a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java @@ -9,8 +9,8 @@ package org.elasticsearch.cluster; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Priority; import org.elasticsearch.core.TimeValue; @@ -23,21 +23,38 @@ public abstract class AckedClusterStateUpdateTask extends ClusterStateUpdateTask implements ClusterStateAckListener { private final ActionListener listener; - private final AckedRequest request; + private final TimeValue ackTimeout; - protected AckedClusterStateUpdateTask(AckedRequest request, ActionListener listener) { - this(Priority.NORMAL, request, listener); + protected AckedClusterStateUpdateTask(AcknowledgedRequest request, ActionListener listener) { + this(Priority.NORMAL, request.masterNodeTimeout(), request.ackTimeout(), listener); + } + + protected AckedClusterStateUpdateTask( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + this(Priority.NORMAL, masterNodeTimeout, ackTimeout, listener); + } + + protected AckedClusterStateUpdateTask( + Priority priority, + AcknowledgedRequest request, + ActionListener listener + ) { + this(priority, request.masterNodeTimeout(), request.ackTimeout(), listener); } @SuppressWarnings("unchecked") protected AckedClusterStateUpdateTask( Priority priority, - AckedRequest request, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, ActionListener listener ) { - super(priority, request.masterNodeTimeout()); + super(priority, masterNodeTimeout); this.listener = (ActionListener) listener; - this.request = request; + this.ackTimeout = ackTimeout; } /** @@ -81,6 +98,6 @@ public void onFailure(Exception e) { * Acknowledgement timeout, maximum time interval to wait for acknowledgements */ public final TimeValue ackTimeout() { - return request.ackTimeout(); + return ackTimeout; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java index 69a51c80839d0..8841b315b0138 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java @@ -15,7 +15,7 @@ * Base class to be used when needing to update the cluster state * Contains the basic fields that are always needed */ -public abstract class ClusterStateUpdateRequest> implements AckedRequest { +public abstract class ClusterStateUpdateRequest> { private TimeValue ackTimeout; private TimeValue masterNodeTimeout; @@ -23,7 +23,6 @@ public abstract class ClusterStateUpdateRequest(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-data-stream [" + request.name + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { // When we're manually creating a data stream (i.e. not an auto creation), we don't need to initialize the failure store @@ -138,13 +142,19 @@ public ClusterState createDataStream( ); } - public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest< - CreateDataStreamClusterStateUpdateRequest> { - - private final boolean performReroute; - private final String name; - private final long startTime; - private final SystemDataStreamDescriptor descriptor; + public record CreateDataStreamClusterStateUpdateRequest( + String name, + long startTime, + @Nullable SystemDataStreamDescriptor systemDataStreamDescriptor, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + boolean performReroute + ) { + public CreateDataStreamClusterStateUpdateRequest { + Objects.requireNonNull(name); + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + } public CreateDataStreamClusterStateUpdateRequest(String name) { this(name, System.currentTimeMillis(), null, TimeValue.ZERO, TimeValue.ZERO, true); @@ -154,42 +164,14 @@ public CreateDataStreamClusterStateUpdateRequest( String name, SystemDataStreamDescriptor systemDataStreamDescriptor, TimeValue masterNodeTimeout, - TimeValue timeout, + TimeValue ackTimeout, boolean performReroute ) { - this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, timeout, performReroute); - } - - public CreateDataStreamClusterStateUpdateRequest( - String name, - long startTime, - SystemDataStreamDescriptor systemDataStreamDescriptor, - TimeValue masterNodeTimeout, - TimeValue timeout, - boolean performReroute - ) { - this.name = name; - this.startTime = startTime; - this.descriptor = systemDataStreamDescriptor; - this.performReroute = performReroute; - masterNodeTimeout(masterNodeTimeout); - ackTimeout(timeout); + this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, ackTimeout, performReroute); } public boolean isSystem() { - return descriptor != null; - } - - public boolean performReroute() { - return performReroute; - } - - public SystemDataStreamDescriptor getSystemDataStreamDescriptor() { - return descriptor; - } - - long getStartTime() { - return startTime; + return systemDataStreamDescriptor != null; } } @@ -238,7 +220,7 @@ static ClusterState createDataStream( boolean initializeFailureStore ) throws Exception { String dataStreamName = request.name; - SystemDataStreamDescriptor systemDataStreamDescriptor = request.getSystemDataStreamDescriptor(); + SystemDataStreamDescriptor systemDataStreamDescriptor = request.systemDataStreamDescriptor(); boolean isSystemDataStreamName = metadataCreateIndexService.getSystemIndices().isSystemDataStream(request.name); assert (isSystemDataStreamName && systemDataStreamDescriptor != null) || (isSystemDataStreamName == false && systemDataStreamDescriptor == null) @@ -287,13 +269,13 @@ static ClusterState createDataStream( if (isSystem) { throw new IllegalArgumentException("Failure stores are not supported on system data streams"); } - String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.getStartTime()); + String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.startTime()); currentState = createFailureStoreIndex( metadataCreateIndexService, "initialize_data_stream", settings, currentState, - request.getStartTime(), + request.startTime(), dataStreamName, template, failureStoreIndexName, @@ -303,7 +285,7 @@ static ClusterState createDataStream( } if (writeIndex == null) { - String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.getStartTime()); + String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.startTime()); currentState = createBackingIndex( metadataCreateIndexService, currentState, @@ -392,7 +374,7 @@ private static ClusterState createBackingIndex( firstBackingIndexName ).dataStreamName(dataStreamName) .systemDataStreamDescriptor(systemDataStreamDescriptor) - .nameResolvedInstant(request.getStartTime()) + .nameResolvedInstant(request.startTime()) .performReroute(request.performReroute()) .setMatchingTemplate(template); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 275c186a1ea85..061aa18dd464a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -296,7 +296,12 @@ private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request, var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-index [" + request.index() + "], cause [" + request.cause() + "]", - new AckedClusterStateUpdateTask(Priority.URGENT, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.URGENT, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java index ced5d4e490478..5d1a037d6bc3e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java @@ -11,13 +11,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateAckListener; import org.elasticsearch.cluster.ClusterStateTaskExecutor; +import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SimpleBatchedAckListenerTaskExecutor; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; @@ -25,7 +28,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.injection.guice.Inject; @@ -33,10 +36,10 @@ import org.elasticsearch.snapshots.SnapshotInProgressException; import org.elasticsearch.snapshots.SnapshotsService; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired; @@ -48,22 +51,19 @@ public class MetadataDeleteIndexService { private static final Logger logger = LogManager.getLogger(MetadataDeleteIndexService.class); - private final Settings settings; - // package private for tests - final ClusterStateTaskExecutor executor; - private final MasterServiceTaskQueue taskQueue; + final ClusterStateTaskExecutor executor; + private final MasterServiceTaskQueue taskQueue; @Inject public MetadataDeleteIndexService(Settings settings, ClusterService clusterService, AllocationService allocationService) { - this.settings = settings; executor = new SimpleBatchedAckListenerTaskExecutor<>() { @Override public Tuple executeTask( - DeleteIndexClusterStateUpdateRequest task, + DeleteIndicesClusterStateUpdateTask task, ClusterState clusterState ) { - return Tuple.tuple(MetadataDeleteIndexService.deleteIndices(clusterState, Sets.newHashSet(task.indices()), settings), task); + return Tuple.tuple(MetadataDeleteIndexService.deleteIndices(clusterState, task.indices, settings), task); } @Override @@ -81,11 +81,64 @@ public ClusterState afterBatchExecution(ClusterState clusterState, boolean clust taskQueue = clusterService.createTaskQueue("delete-index", Priority.URGENT, executor); } - public void deleteIndices(final DeleteIndexClusterStateUpdateRequest request) { - if (request.indices() == null || request.indices().length == 0) { - throw new IllegalArgumentException("Index name is required"); + public void deleteIndices( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + Set indices, + ActionListener listener + ) { + if (indices == null || indices.isEmpty()) { + throw new IllegalArgumentException("Indices are required"); + } + taskQueue.submitTask( + "delete-index " + indices, + new DeleteIndicesClusterStateUpdateTask(indices, ackTimeout, listener), + masterNodeTimeout + ); + } + + // package private for tests + static class DeleteIndicesClusterStateUpdateTask implements ClusterStateTaskListener, ClusterStateAckListener { + + private final Set indices; + private final TimeValue ackTimeout; + private final ActionListener listener; + + DeleteIndicesClusterStateUpdateTask(Set indices, TimeValue ackTimeout, ActionListener listener) { + this.indices = Objects.requireNonNull(indices); + this.ackTimeout = Objects.requireNonNull(ackTimeout); + this.listener = Objects.requireNonNull(listener); + } + + @Override + public boolean mustAck(DiscoveryNode discoveryNode) { + return true; + } + + @Override + public void onAllNodesAcked() { + listener.onResponse(AcknowledgedResponse.TRUE); + } + + @Override + public void onAckFailure(Exception e) { + listener.onResponse(AcknowledgedResponse.FALSE); + } + + @Override + public void onAckTimeout() { + listener.onResponse(AcknowledgedResponse.FALSE); + } + + @Override + public TimeValue ackTimeout() { + return ackTimeout; + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); } - taskQueue.submitTask("delete-index " + Arrays.toString(request.indices()), request, request.masterNodeTimeout()); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 130a38f39b11a..f926e2b6ebf35 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -282,7 +282,7 @@ public boolean mustAck(DiscoveryNode discoveryNode) { @Override public void onAllNodesAcked() { - listener.onResponse(IndicesAliasesResponse.build(request.getActionResults())); + listener.onResponse(IndicesAliasesResponse.build(request.actionResults())); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 1c93a13583e94..aee60c3eda57f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener; import org.elasticsearch.cluster.service.ClusterService; @@ -40,6 +39,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -105,7 +105,12 @@ public void migrateToDataStream( var delegate = new AllocationActionListener<>(listener, threadContext); submitUnbatchedTask( "migrate-to-data-stream [" + request.aliasName + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -278,15 +283,11 @@ static void validateBackingIndices(ClusterState currentState, String dataStreamN } } - @SuppressWarnings("rawtypes") - public static final class MigrateToDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { - - private final String aliasName; - - public MigrateToDataStreamClusterStateUpdateRequest(String aliasName, TimeValue masterNodeTimeout, TimeValue timeout) { - this.aliasName = aliasName; - masterNodeTimeout(masterNodeTimeout); - ackTimeout(timeout); + public record MigrateToDataStreamClusterStateUpdateRequest(String aliasName, TimeValue masterNodeTimeout, TimeValue ackTimeout) { + public MigrateToDataStreamClusterStateUpdateRequest { + Objects.requireNonNull(aliasName); + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 8a69c66bf8375..3fb3c182f89cd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -36,7 +36,6 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -46,6 +45,7 @@ import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.expectValueToken; /** * Generates the shard id for {@code (id, routing)} pairs. @@ -53,6 +53,7 @@ public abstract class IndexRouting { static final NodeFeature BOOLEAN_ROUTING_PATH = new NodeFeature("routing.boolean_routing_path"); + static final NodeFeature MULTI_VALUE_ROUTING_PATH = new NodeFeature("routing.multi_value_routing_path"); /** * Build the routing from {@link IndexMetadata}. @@ -302,7 +303,13 @@ public String createId(Map flat, byte[] suffix) { Builder b = builder(); for (Map.Entry e : flat.entrySet()) { if (isRoutingPath.test(e.getKey())) { - b.hashes.add(new NameAndHash(new BytesRef(e.getKey()), hash(new BytesRef(e.getValue().toString())))); + if (e.getValue() instanceof List listValue) { + for (Object v : listValue) { + b.addHash(e.getKey(), new BytesRef(v.toString())); + } + } else { + b.addHash(e.getKey(), new BytesRef(e.getValue().toString())); + } } } return b.createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); @@ -337,7 +344,7 @@ public class Builder { public void addMatching(String fieldName, BytesRef string) { if (isRoutingPath.test(fieldName)) { - hashes.add(new NameAndHash(new BytesRef(fieldName), hash(string))); + addHash(fieldName, string); } } @@ -358,6 +365,13 @@ private void extractObject(@Nullable String path, XContentParser source) throws } } + private void extractArray(@Nullable String path, XContentParser source) throws IOException { + while (source.currentToken() != Token.END_ARRAY) { + expectValueToken(source.currentToken(), source); + extractItem(path, source); + } + } + private void extractItem(String path, XContentParser source) throws IOException { switch (source.currentToken()) { case START_OBJECT: @@ -368,7 +382,12 @@ private void extractItem(String path, XContentParser source) throws IOException case VALUE_STRING: case VALUE_NUMBER: case VALUE_BOOLEAN: - hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); + addHash(path, new BytesRef(source.text())); + source.nextToken(); + break; + case START_ARRAY: + source.nextToken(); + extractArray(path, source); source.nextToken(); break; case VALUE_NULL: @@ -377,28 +396,24 @@ private void extractItem(String path, XContentParser source) throws IOException default: throw new ParsingException( source.getTokenLocation(), - "Routing values must be strings but found [{}]", + "Cannot extract routing path due to unexpected token [{}]", source.currentToken() ); } } + private void addHash(String path, BytesRef value) { + hashes.add(new NameAndHash(new BytesRef(path), hash(value), hashes.size())); + } + private int buildHash(IntSupplier onEmpty) { - Collections.sort(hashes); - Iterator itr = hashes.iterator(); - if (itr.hasNext() == false) { + if (hashes.isEmpty()) { return onEmpty.getAsInt(); } - NameAndHash prev = itr.next(); - int hash = hash(prev.name) ^ prev.hash; - while (itr.hasNext()) { - NameAndHash next = itr.next(); - if (prev.name.equals(next.name)) { - throw new IllegalArgumentException("Duplicate routing dimension for [" + next.name + "]"); - } - int thisHash = hash(next.name) ^ next.hash; - hash = 31 * hash + thisHash; - prev = next; + Collections.sort(hashes); + int hash = 0; + for (NameAndHash nah : hashes) { + hash = 31 * hash + (hash(nah.name) ^ nah.hash); } return hash; } @@ -459,10 +474,13 @@ private String error(String operation) { } } - private record NameAndHash(BytesRef name, int hash) implements Comparable { + private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { @Override public int compareTo(NameAndHash o) { - return name.compareTo(o.name); + int i = name.compareTo(o.name); + if (i != 0) return i; + // ensures array values are in the order as they appear in the source + return Integer.compare(order, o.order); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java index 8885410bd0530..f8028ce7f9d68 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java @@ -18,6 +18,6 @@ public class RoutingFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(IndexRouting.BOOLEAN_ROUTING_PATH); + return Set.of(IndexRouting.BOOLEAN_ROUTING_PATH, IndexRouting.MULTI_VALUE_ROUTING_PATH); } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java index 893c6b9714eaa..0aacc466cce7a 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java @@ -177,6 +177,38 @@ public void writeWithSizePrefix(Writeable writeable) throws IOException { } } + // overridden with some code duplication the same way other write methods in this class are overridden to bypass StreamOutput's + // intermediary buffers + @Override + public void writeString(String str) throws IOException { + final int currentPageOffset = this.currentPageOffset; + final int charCount = str.length(); + // maximum serialized length is 3 bytes per char + 5 bytes for the longest possible vint + if (charCount * 3 + 5 > (pageSize - currentPageOffset)) { + super.writeString(str); + return; + } + BytesRef currentPage = pages.get(pageIndex).v(); + int off = currentPage.offset + currentPageOffset; + byte[] buffer = currentPage.bytes; + // mostly duplicated from StreamOutput.writeString to to get more reliable compilation of this very hot loop + int offset = off + putVInt(buffer, charCount, off); + for (int i = 0; i < charCount; i++) { + final int c = str.charAt(i); + if (c <= 0x007F) { + buffer[offset++] = ((byte) c); + } else if (c > 0x07FF) { + buffer[offset++] = ((byte) (0xE0 | c >> 12 & 0x0F)); + buffer[offset++] = ((byte) (0x80 | c >> 6 & 0x3F)); + buffer[offset++] = ((byte) (0x80 | c >> 0 & 0x3F)); + } else { + buffer[offset++] = ((byte) (0xC0 | c >> 6 & 0x1F)); + buffer[offset++] = ((byte) (0x80 | c >> 0 & 0x3F)); + } + } + this.currentPageOffset = offset - currentPage.offset; + } + @Override public void flush() { // nothing to do diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index 8bca42e60c7b6..1d84623ba06e4 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -2324,8 +2324,6 @@ static DateFormatter forPattern(String input) { } else if (FormatNames.STRICT_YEAR_MONTH_DAY.matches(input)) { return STRICT_YEAR_MONTH_DAY; } else { - DateUtils.checkTextualDateFormats(input); - try { return newDateFormatter( input, diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index 849a64f307f7a..9f642734ba832 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -11,9 +11,6 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.Predicates; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.logging.LogManager; import java.time.Clock; import java.time.Duration; @@ -23,8 +20,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.function.Predicate; -import java.util.regex.Pattern; import static java.util.Map.entry; import static org.elasticsearch.common.time.DateUtilsRounding.getMonthOfYear; @@ -388,16 +383,4 @@ public static ZonedDateTime nowWithMillisResolution(Clock clock) { Clock millisResolutionClock = Clock.tick(clock, Duration.ofMillis(1)); return ZonedDateTime.now(millisResolutionClock); } - - // check for all textual fields, and localized zone offset - private static final Predicate CONTAINS_CHANGING_TEXT_SPECIFIERS = System.getProperty("java.locale.providers", "") - .contains("COMPAT") ? Pattern.compile("[EcGaO]|MMM|LLL|eee|ccc|QQQ|ZZZZ").asPredicate() : Predicates.never(); - - @UpdateForV9 // this can be removed, we will only use CLDR on v9 - static void checkTextualDateFormats(String format) { - if (CONTAINS_CHANGING_TEXT_SPECIFIERS.test(format)) { - LogManager.getLogger(DateFormatter.class) - .warn("Date format [{}] contains textual field specifiers that could change in JDK 23", format); - } - } } diff --git a/server/src/main/java/org/elasticsearch/common/time/IsoCalendarDataProvider.java b/server/src/main/java/org/elasticsearch/common/time/IsoCalendarDataProvider.java deleted file mode 100644 index 1093292b388e3..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/time/IsoCalendarDataProvider.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package org.elasticsearch.common.time; - -import java.util.Calendar; -import java.util.Locale; -import java.util.spi.CalendarDataProvider; - -public class IsoCalendarDataProvider extends CalendarDataProvider { - - @Override - public int getFirstDayOfWeek(Locale locale) { - return Calendar.MONDAY; - } - - @Override - public int getMinimalDaysInFirstWeek(Locale locale) { - return 4; - } - - @Override - public Locale[] getAvailableLocales() { - return new Locale[] { Locale.ROOT }; - } -} diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java index cd7cf5ddbd893..6390e62f9758f 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java @@ -72,6 +72,20 @@ public static void ensureExpectedToken(Token expected, Token actual, XContentPar } } + /** + * Makes sure the provided token {@linkplain Token#isValue() is a value type} + * + * @throws ParsingException if the token is not a value type + */ + public static void expectValueToken(Token actual, XContentParser parser) { + if (actual.isValue() == false) { + throw new ParsingException( + parser.getTokenLocation(), + String.format(Locale.ROOT, "Failed to parse object: expecting value token but found [%s]", actual) + ); + } + } + private static ParsingException parsingException(XContentParser parser, Token expected, Token actual) { return new ParsingException( parser.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java b/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java index 77454c5686538..ccf375aec60a5 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java +++ b/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java @@ -33,6 +33,24 @@ public interface HttpPreRequest { */ String uri(); + /** + * The uri without the query string. + */ + default String rawPath() { + String uri = uri(); + final int index = uri.indexOf('?'); + if (index >= 0) { + return uri.substring(0, index); + } else { + final int index2 = uri.indexOf('#'); + if (index2 >= 0) { + return uri.substring(0, index2); + } else { + return uri; + } + } + } + /** * Get all of the headers and values associated with the HTTP headers. * Modifications of this map are not supported. diff --git a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java index 1a90f5f110376..462490a7fceb7 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.analysis; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.ar.ArabicAnalyzer; import org.apache.lucene.analysis.bg.BulgarianAnalyzer; @@ -67,6 +69,7 @@ import java.security.AccessControlException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -78,6 +81,7 @@ public class Analysis { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(Analysis.class); + private static final Logger logger = LogManager.getLogger(Analysis.class); public static void checkForDeprecatedVersion(String name, Settings settings) { String sVersion = settings.get("version"); @@ -267,12 +271,14 @@ public static List getWordList( Settings settings, String settingPath, String settingList, + String settingLenient, boolean removeComments, boolean checkDuplicate ) { + boolean deduplicateDictionary = settings.getAsBoolean(settingLenient, false); final List ruleList = getWordList(env, settings, settingPath, settingList, removeComments); if (ruleList != null && ruleList.isEmpty() == false && checkDuplicate) { - checkDuplicateRules(ruleList); + return deDuplicateRules(ruleList, deduplicateDictionary == false); } return ruleList; } @@ -288,24 +294,36 @@ public static List getWordList( * If the addition to the HashSet returns false, it means that item was already present in the set, indicating a duplicate. * In such a case, an IllegalArgumentException is thrown specifying the duplicate term and the line number in the original list. * + * Optionally the function will return the deduplicated list + * * @param ruleList The list of rules to check for duplicates. * @throws IllegalArgumentException If a duplicate rule is found. */ - private static void checkDuplicateRules(List ruleList) { - Set dup = new HashSet<>(); - int lineNum = 0; - for (String line : ruleList) { - // ignore comments + private static List deDuplicateRules(List ruleList, boolean failOnDuplicate) { + Set duplicateKeys = new HashSet<>(); + List deduplicatedList = new ArrayList<>(); + for (int lineNum = 0; lineNum < ruleList.size(); lineNum++) { + String line = ruleList.get(lineNum); + // ignore lines beginning with # as those are comments if (line.startsWith("#") == false) { String[] values = CSVUtil.parse(line); - if (dup.add(values[0]) == false) { - throw new IllegalArgumentException( - "Found duplicate term [" + values[0] + "] in user dictionary " + "at line [" + lineNum + "]" - ); + if (duplicateKeys.add(values[0]) == false) { + if (failOnDuplicate) { + throw new IllegalArgumentException( + "Found duplicate term [" + values[0] + "] in user dictionary " + "at line [" + (lineNum + 1) + "]" + ); + } else { + logger.warn("Ignoring duplicate term [" + values[0] + "] in user dictionary " + "at line [" + (lineNum + 1) + "]"); + } + } else { + deduplicatedList.add(line); } + } else { + deduplicatedList.add(line); } - ++lineNum; } + + return Collections.unmodifiableList(deduplicatedList); } private static List loadWordList(Path path, boolean removeComments) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java index 7e5494007b369..71d9768ac5ff7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java @@ -533,7 +533,7 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOExcepti int i, docValueCount; @Override - public long nextValue() throws IOException { + public long nextValue() { return ords[i++]; } @@ -543,7 +543,7 @@ public int docValueCount() { } @Override - public boolean advanceExact(int target) throws IOException { + public boolean advanceExact(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java index 597fabc5418f2..d46f8a3f20a4f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java @@ -10,13 +10,9 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.search.DocIdSetIterator; - -import java.io.IOException; /** - * Base implementation that throws an {@link IOException} for the - * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * Base implementation. This impl is safe to use for sorting and * aggregations, which only use {@link #advanceExact(int)} and * {@link #binaryValue()}. */ @@ -28,12 +24,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java index e6925dde709b8..fca1adc137b71 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java @@ -23,12 +23,12 @@ public abstract class AbstractNumericDocValues extends NumericDocValues { @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java index e5e75cae041e4..05bb9a8fe74c4 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java @@ -23,12 +23,12 @@ public abstract class AbstractSortedDocValues extends SortedDocValues { @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java index a1c3a054fdd27..4c17715ec55f8 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java @@ -33,7 +33,7 @@ public int nextDoc() throws IOException { } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java index 90d577c21568e..c040a2e8dc2d0 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java @@ -28,12 +28,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java index 2684b40f4e442..076b2fecf4bef 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java @@ -36,12 +36,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java index b618268d9a5b0..1a612bbe724f9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java @@ -66,9 +66,5 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java index 3b8cc9956442a..742e0a0f95078 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java @@ -43,7 +43,7 @@ public DoubleScriptFieldData build(IndexFieldDataCache cache, CircuitBreakerServ } private final String fieldName; - DoubleFieldScript.LeafFactory leafFactory; + final DoubleFieldScript.LeafFactory leafFactory; private final ToScriptFieldFactory toScriptFieldFactory; private DoubleScriptFieldData( diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index b090354b08459..6c8d52ac18280 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -42,12 +42,12 @@ public static SortedBinaryDocValues emptySortedBinary() { public static NumericDoubleValues emptyNumericDouble() { return new NumericDoubleValues() { @Override - public boolean advanceExact(int doc) throws IOException { + public boolean advanceExact(int doc) { return false; } @Override - public double doubleValue() throws IOException { + public double doubleValue() { throw new UnsupportedOperationException(); } @@ -243,14 +243,6 @@ public static BinaryDocValues unwrapSingleton(SortedBinaryDocValues values) { return null; } - /** - * Returns whether the provided values *might* be multi-valued. There is no - * guarantee that this method will return {@code false} in the single-valued case. - */ - public static boolean isMultiValued(SortedSetDocValues values) { - return DocValues.unwrapSingleton(values) == null; - } - /** * Return a {@link String} representation of the provided values. That is * typically used for scripts or for the `map` execution mode of terms aggs. @@ -634,7 +626,7 @@ public boolean advanceExact(int target) throws IOException { } @Override - public long longValue() throws IOException { + public long longValue() { return value; } }; @@ -661,7 +653,7 @@ public boolean advanceExact(int target) throws IOException { } @Override - public double doubleValue() throws IOException { + public double doubleValue() { return value; } }; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java index 25c28a99a600c..c9aed46377c3f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java @@ -67,6 +67,6 @@ public int docValueCount() { public long nextValue() { int lat = GeoEncodingUtils.encodeLatitude(script.lats()[cursor]); int lon = GeoEncodingUtils.encodeLongitude(script.lons()[cursor++]); - return Long.valueOf((((long) lat) << 32) | (lon & 0xFFFFFFFFL)); + return (((long) lat) << 32) | (lon & 0xFFFFFFFFL); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java index 1ffb0deca8a3d..e6bd6c3e59656 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java @@ -103,10 +103,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 902eba0f7d5d8..65eb724549bfc 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -20,7 +20,6 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.Closeable; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -90,7 +89,6 @@ public synchronized void clearField(final String fieldName) { * Returns fielddata for the provided field type, given the provided fully qualified index name, while also making * a {@link SearchLookup} supplier available that is required for runtime fields. */ - @SuppressWarnings("unchecked") public > IFD getForField(MappedFieldType fieldType, FieldDataContext fieldDataContext) { return getFromBuilder(fieldType, fieldType.fielddataBuilder(fieldDataContext)); } @@ -134,7 +132,7 @@ public void setListener(IndexFieldDataCache.Listener listener) { } @Override - public void close() throws IOException { + public void close() { clear(); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index e1e8dad62daed..289f1dd6abd25 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -198,20 +198,16 @@ private XFieldComparatorSource comparatorSource( MultiValueMode sortMode, Nested nested ) { - switch (targetNumericType) { - case HALF_FLOAT: - case FLOAT: - return new FloatValuesComparatorSource(this, missingValue, sortMode, nested); - case DOUBLE: - return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); - case DATE: - return dateComparatorSource(missingValue, sortMode, nested); - case DATE_NANOSECONDS: - return dateNanosComparatorSource(missingValue, sortMode, nested); - default: + return switch (targetNumericType) { + case HALF_FLOAT, FLOAT -> new FloatValuesComparatorSource(this, missingValue, sortMode, nested); + case DOUBLE -> new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); + case DATE -> dateComparatorSource(missingValue, sortMode, nested); + case DATE_NANOSECONDS -> dateNanosComparatorSource(missingValue, sortMode, nested); + default -> { assert targetNumericType.isFloatingPoint() == false; - return new LongValuesComparatorSource(this, missingValue, sortMode, nested, targetNumericType); - } + yield new LongValuesComparatorSource(this, missingValue, sortMode, nested, targetNumericType); + } + }; } protected XFieldComparatorSource dateComparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java index 2c48eb28e7fe5..e42db267920ed 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java @@ -32,6 +32,9 @@ public interface LeafFieldData extends Accountable, Releasable { */ SortedBinaryDocValues getBytesValues(); + @Override + default void close() {} + /** * Return a formatted representation of the values */ diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java index 2ad11aa771a5c..7f05ca0e50f92 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java @@ -20,8 +20,6 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import java.io.IOException; - public final class LongScriptFieldData extends IndexNumericFieldData { public static class Builder implements IndexFieldData.Builder { @@ -79,7 +77,7 @@ public LongScriptLeafFieldData load(LeafReaderContext context) { } @Override - public LongScriptLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + public LongScriptLeafFieldData loadDirect(LeafReaderContext context) { return new LongScriptLeafFieldData(new LongScriptDocValues(leafFactory.newInstance(context)), toScriptFieldFactory); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java index 449b3b9ced777..06deda44dc364 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java @@ -45,7 +45,6 @@ public GeoPoint nextValue() throws IOException { /** * Returns a single-valued view of the {@link MultiPointValues} if possible, otherwise null. */ - @Override protected GeoPointValues getPointValues() { final NumericDocValues singleton = DocValues.unwrapSingleton(numericValues); return singleton != null ? new GeoPointValues(singleton) : null; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java index f99683ebe0831..854f06d9b606a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java @@ -53,8 +53,4 @@ public int docValueCount() { */ public abstract T nextValue() throws IOException; - /** - * Returns a single-valued view of the {@link MultiPointValues} if possible, otherwise null. - */ - protected abstract PointValues getPointValues(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java b/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java index 15817ae513146..14a1f287eb500 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java @@ -52,7 +52,7 @@ public RamAccountingTermsEnum( * Always accept the term. */ @Override - protected AcceptStatus accept(BytesRef term) throws IOException { + protected AcceptStatus accept(BytesRef term) { return AcceptStatus.YES; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java index d5933c7fab4d2..02f5de7e3c8ca 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java @@ -25,7 +25,7 @@ public abstract class SortingNumericDocValues extends SortedNumericDocValues { protected long[] values; protected int valuesCursor; private final Sorter sorter; - private LongConsumer circuitBreakerConsumer; + private final LongConsumer circuitBreakerConsumer; protected SortingNumericDocValues() { this(l -> {}); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java index d9f6cefeba38c..bb4eb3f0bebce 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java @@ -132,11 +132,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } - @Override public SortedBinaryDocValues getBytesValues() { throw new IllegalArgumentException("not supported for source fallback"); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java index a4ff63ae1e04e..bfb5c58cac3af 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java @@ -61,7 +61,7 @@ protected SourceValueFetcherMultiGeoPointIndexFieldData( } @Override - public SourceValueFetcherMultiGeoPointLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherMultiGeoPointLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherMultiGeoPointLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java index 3f26a6c71197d..e24a6c5251f8f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java @@ -62,7 +62,7 @@ protected SourceValueFetcherSortedBinaryIndexFieldData( } @Override - public SourceValueFetcherSortedBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherSortedBinaryLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedBinaryLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -127,7 +127,7 @@ public int docValueCount() { } @Override - public BytesRef nextValue() throws IOException { + public BytesRef nextValue() { assert iterator.hasNext(); return iterator.next(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java index c28890cb82671..56937d47b794d 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java @@ -59,7 +59,7 @@ protected SourceValueFetcherSortedBooleanIndexFieldData( } @Override - public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedBooleanLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -129,7 +129,7 @@ public int docValueCount() { } @Override - public long nextValue() throws IOException { + public long nextValue() { assert iteratorIndex < trueCount + falseCount; return iteratorIndex++ < falseCount ? 0L : 1L; } @@ -140,12 +140,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException("not supported for source fallback"); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException("not supported for source fallback"); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java index 2df4917390354..f4bc2216217a7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java @@ -61,7 +61,7 @@ protected SourceValueFetcherSortedDoubleIndexFieldData( } @Override - public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedDoubleLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -128,7 +128,7 @@ public int docValueCount() { } @Override - public double nextValue() throws IOException { + public double nextValue() { assert iterator.hasNext(); return iterator.next(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java index cf4cbf93b8e8d..ce1dff33e80ce 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java @@ -62,7 +62,7 @@ protected SourceValueFetcherSortedNumericIndexFieldData( } @Override - public SourceValueFetcherSortedNumericLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherSortedNumericLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedNumericLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -129,7 +129,7 @@ public int docValueCount() { } @Override - public long nextValue() throws IOException { + public long nextValue() { assert iterator.hasNext(); return iterator.next(); } @@ -140,12 +140,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException("not supported for source fallback"); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException("not supported for source fallback"); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java index 6d3d5b988c9f9..a0c4efe9ef6c1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java @@ -104,9 +104,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} - @Override public SortedBinaryDocValues getBytesValues() { throw new IllegalArgumentException("not supported for source fallback"); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java index 69e348956822e..550833c396430 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java @@ -75,7 +75,7 @@ public int docValueCount() { } @Override - public BytesRef nextValue() throws IOException { + public BytesRef nextValue() { assert current < docValueCount; return sorted.get(current++); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 459075cfcc5b1..f1e4ba95e76fe 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -54,7 +54,7 @@ public SortField.Type reducedType() { return SortField.Type.FLOAT; } - private NumericDoubleValues getNumericDocValues(LeafReaderContext context, float missingValue) throws IOException { + private NumericDoubleValues getNumericDocValues(LeafReaderContext context, double missingValue) throws IOException { final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues(); if (nested == null) { return FieldData.replaceMissing(sortMode.select(values), missingValue); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index e857a715ee657..def1c40e1e6f0 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -111,8 +111,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} }; subs[i] = atomicFD[i].getOrdinalsValues(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java index 2ffc8527b179a..5b3e523a10f41 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java @@ -76,7 +76,7 @@ public IndexOrdinalsFieldData newConsumer(DirectoryReader source) { } @Override - public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) { throw new IllegalStateException("loadDirect(LeafReaderContext) should not be called in this context"); } @@ -86,7 +86,7 @@ public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { } @Override - public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) throws Exception { + public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) { return this; } @@ -179,7 +179,7 @@ private TermsEnum[] getOrLoadTermsEnums() { } @Override - public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) { return load(context); } @@ -189,7 +189,7 @@ public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { } @Override - public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) throws Exception { + public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) { return this; } @@ -258,8 +258,6 @@ public Collection getChildResources() { return segmentAfd[context.ord].getChildResources(); } - @Override - public void close() {} }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index d39e811ae49f4..e1b66ec379d8e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -16,7 +16,6 @@ import org.apache.lucene.util.packed.PagedGrowableWriter; import java.io.Closeable; -import java.io.IOException; import java.util.Arrays; /** @@ -369,7 +368,7 @@ public int maxDoc() { * Closes this builder and release all resources. */ @Override - public void close() throws IOException { + public void close() { ordinals = null; } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java index d609796d58c83..63fa0e21c6cb6 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java @@ -68,8 +68,4 @@ public BytesRef nextValue() throws IOException { }; } - @Override - public void close() { - // no-op - } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java index 5227a4bed1f1c..5038117a5d554 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java @@ -83,13 +83,7 @@ public LeafOrdinalsFieldData load(LeafReaderContext context) { try { return cache.load(context, this); } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else if (e instanceof ExecutionException && e.getCause() instanceof ElasticsearchException) { - throw (ElasticsearchException) e.getCause(); - } else { - throw new ElasticsearchException(e); - } + throw handleCacheLoadException(e); } } @@ -131,13 +125,7 @@ private IndexOrdinalsFieldData loadGlobalInternal(DirectoryReader indexReader) { try { return cache.load(indexReader, this); } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else if (e instanceof ExecutionException && e.getCause() instanceof ElasticsearchException) { - throw (ElasticsearchException) e.getCause(); - } else { - throw new ElasticsearchException(e); - } + throw handleCacheLoadException(e); } } @@ -157,6 +145,16 @@ public boolean supportsGlobalOrdinalsMapping() { return false; } + private static ElasticsearchException handleCacheLoadException(Exception e) { + if (e instanceof ElasticsearchException ese) { + return ese; + } + if (e instanceof ExecutionException && e.getCause() instanceof ElasticsearchException ese) { + throw ese; + } + throw new ElasticsearchException(e); + } + /** * A {@code PerValueEstimator} is a sub-class that can be used to estimate * the memory overhead for loading the data. Each field data diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java index 8dc30f2646a8a..d1280040f085f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java @@ -43,9 +43,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} - @Override public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java index 22bceca37fb6c..54f1ecac89f8e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java @@ -47,11 +47,6 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { return new DelegateDocValuesField(new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(getBytesValues())), name); } - @Override - public void close() { - // no-op - } - @Override public long ramBytesUsed() { return 0; // unknown diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java index f414f800bf809..0d90ef9a41071 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java @@ -65,7 +65,7 @@ public BinaryDVLeafFieldData load(LeafReaderContext context) { } @Override - public BinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public BinaryDVLeafFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java index 2f0f7254f1265..f5d4bf837f062 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java @@ -75,7 +75,7 @@ public BytesBinaryDVLeafFieldData load(LeafReaderContext context) { } @Override - public BytesBinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public BytesBinaryDVLeafFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java index 85adc2b5c5012..8e15e6fc8e7f1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java @@ -113,9 +113,6 @@ public int docID() { return DocValues.singleton(sortedValues); } - @Override - public void close() {} - } private final ConstantLeafFieldData atomicFieldData; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/FormattedSortedNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/FormattedSortedNumericDocValues.java new file mode 100644 index 0000000000000..da64520c7c86c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/FormattedSortedNumericDocValues.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; + +public final class FormattedSortedNumericDocValues implements FormattedDocValues { + private final SortedNumericDocValues values; + private final DocValueFormat format; + + public FormattedSortedNumericDocValues(SortedNumericDocValues values, DocValueFormat format) { + this.values = values; + this.format = format; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + return format.format(values.nextValue()); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java index f303c61c86b6e..64e6dd223caf4 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java @@ -31,11 +31,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public SortedNumericDocValues getSortedNumericDocValues() { try { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java index 40f934746126a..cbf7d76f24137 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java @@ -43,7 +43,7 @@ public LeafPointFieldData load(LeafReaderContext context) { } @Override - public LeafPointFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafPointFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 144dc9220110c..e870f7af22562 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -60,7 +60,4 @@ public Object nextValue() throws IOException { }; } - @Override - public void close() {} - } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index 5c849fa1efaa6..bad40d2c51943 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.fielddata.plain; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.LeafNumericFieldData; @@ -17,8 +16,6 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.DocValueFormat; -import java.io.IOException; - /** * Specialization of {@link LeafNumericFieldData} for integers. */ @@ -47,25 +44,7 @@ public final SortedNumericDoubleValues getDoubleValues() { @Override public FormattedDocValues getFormattedValues(DocValueFormat format) { - SortedNumericDocValues values = getLongValues(); - return new FormattedDocValues() { - @Override - public boolean advanceExact(int docId) throws IOException { - return values.advanceExact(docId); - } - - @Override - public int docValueCount() throws IOException { - return values.docValueCount(); - } - - @Override - public Object nextValue() throws IOException { - return format.format(values.nextValue()); - } - }; + return new FormattedSortedNumericDocValues(getLongValues(), format); } - @Override - public void close() {} } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java index 1dc2203e04c7d..d0b77c70eb0cf 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java @@ -38,9 +38,6 @@ public PagedBytesLeafFieldData( this.ordinals = ordinals; } - @Override - public void close() {} - @Override public long ramBytesUsed() { long size = ordinals.ramBytesUsed(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java index 151ecb7d376e3..ae20100f4c1f7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java @@ -112,7 +112,7 @@ public NumericType getNumericType() { } @Override - public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafNumericFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index d72ca60bb9786..98fd4e27f9422 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -153,7 +153,7 @@ public NumericType getNumericType() { } @Override - public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafNumericFieldData loadDirect(LeafReaderContext context) { return load(context); } @@ -206,25 +206,9 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { @Override public FormattedDocValues getFormattedValues(DocValueFormat format) { - DocValueFormat nanosFormat = DocValueFormat.withNanosecondResolution(format); - SortedNumericDocValues values = getLongValuesAsNanos(); - return new FormattedDocValues() { - @Override - public boolean advanceExact(int docId) throws IOException { - return values.advanceExact(docId); - } - - @Override - public int docValueCount() throws IOException { - return values.docValueCount(); - } - - @Override - public Object nextValue() throws IOException { - return nanosFormat.format(values.nextValue()); - } - }; + return new FormattedSortedNumericDocValues(getLongValuesAsNanos(), DocValueFormat.withNanosecondResolution(format)); } + } /** @@ -264,4 +248,5 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { return toScriptFieldFactory.getScriptFieldFactory(getLongValues(), name); } } + } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java index 9b08c319ecd98..d4682e680a54a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java @@ -40,9 +40,6 @@ public SortedSetDocValues getOrdinalsValues() { } } - @Override - public void close() {} - @Override public long ramBytesUsed() { return 0; // unknown diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java index 954f2db34f4ba..7c8b058cb01f1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java @@ -82,7 +82,7 @@ public BucketedSort newBucketedSort( } @Override - public StringBinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public StringBinaryDVLeafFieldData loadDirect(LeafReaderContext context) { return load(context); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 4d1b68214eddb..c2970d8716147 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -123,6 +123,7 @@ public int get() { private Field version; private final SeqNoFieldMapper.SequenceIDFields seqID; private final Set fieldsAppliedFromTemplates; + /** * Fields that are copied from values of other fields via copy_to. * This per-document state is needed since it is possible @@ -453,26 +454,16 @@ public boolean isFieldAppliedFromTemplate(String name) { public void markFieldAsCopyTo(String fieldName) { copyToFields.add(fieldName); - if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite() == false) { - /* - Mark this field as containing copied data meaning it should not be present - in synthetic _source (to be consistent with stored _source). - Ignored source values take precedence over standard synthetic source implementation - so by adding this nothing entry we "disable" field in synthetic source. - Otherwise, it would be constructed f.e. from doc_values which leads to duplicate values - in copied field after reindexing. - - Note that this applies to fields that are copied from fields using ignored source themselves - and therefore we don't check for canAddIgnoredField(). - */ - ignoredFieldValues.add(IgnoredSourceFieldMapper.NameValue.fromContext(this, fieldName, XContentDataHelper.nothing())); - } } public boolean isCopyToDestinationField(String name) { return copyToFields.contains(name); } + public Set getCopyToFields() { + return copyToFields; + } + /** * Add a new mapper dynamically created while parsing. * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 7b926b091b3a2..d57edb757ba10 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -26,9 +26,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -113,6 +112,10 @@ NameValue cloneWithValue(BytesRef value) { assert value() == null; return new NameValue(name, parentOffset, value, doc); } + + boolean hasValue() { + return XContentDataHelper.isDataPresent(value); + } } static final class IgnoredValuesFieldMapperType extends StringFieldType { @@ -147,11 +150,38 @@ protected String contentType() { @Override public void postParse(DocumentParserContext context) { // Ignored values are only expected in synthetic mode. - assert context.getIgnoredFieldValues().isEmpty() || context.mappingLookup().isSourceSynthetic(); - List ignoredFieldValues = new ArrayList<>(context.getIgnoredFieldValues()); - // ensure consistent ordering when retrieving synthetic source - Collections.sort(ignoredFieldValues, Comparator.comparing(NameValue::name)); - for (NameValue nameValue : ignoredFieldValues) { + if (context.mappingLookup().isSourceSynthetic() == false) { + assert context.getIgnoredFieldValues().isEmpty(); + return; + } + + Collection ignoredValuesToWrite = context.getIgnoredFieldValues(); + if (context.getCopyToFields().isEmpty() == false && indexSettings.getSkipIgnoredSourceWrite() == false) { + /* + Mark fields as containing copied data meaning they should not be present + in synthetic _source (to be consistent with stored _source). + Ignored source values take precedence over standard synthetic source implementation + so by adding the `XContentDataHelper.voidValue()` entry we disable the field in synthetic source. + Otherwise, it would be constructed f.e. from doc_values which leads to duplicate values + in copied field after reindexing. + */ + var mutableList = new ArrayList<>(ignoredValuesToWrite); + for (String copyToField : context.getCopyToFields()) { + ObjectMapper parent = context.parent().findParentMapper(copyToField); + if (parent == null) { + // There are scenarios when this can happen: + // 1. all values of the field that is the source of copy_to are null + // 2. copy_to points at a field inside a disabled object + // 3. copy_to points at dynamic field which is not yet applied to mapping, we will process it properly on re-parse. + continue; + } + int offset = parent.isRoot() ? 0 : parent.fullPath().length() + 1; + mutableList.add(new IgnoredSourceFieldMapper.NameValue(copyToField, offset, XContentDataHelper.voidValue(), context.doc())); + } + ignoredValuesToWrite = mutableList; + } + + for (NameValue nameValue : ignoredValuesToWrite) { nameValue.doc().add(new StoredField(NAME, encode(nameValue))); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 0f224c98241db..d18c3283ef909 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -40,7 +40,8 @@ public Set getFeatures() { Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE, SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT, SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX, - FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT + FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT, + SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index e4ce38d6cec0b..f9c854749e885 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -808,6 +808,42 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep } + ObjectMapper findParentMapper(String leafFieldPath) { + var pathComponents = leafFieldPath.split("\\."); + int startPathComponent = 0; + + ObjectMapper current = this; + String pathInCurrent = leafFieldPath; + + while (current != null) { + if (current.mappers.containsKey(pathInCurrent)) { + return current; + } + + // Go one level down if possible + var parent = current; + current = null; + + var childMapperName = new StringBuilder(); + for (int i = startPathComponent; i < pathComponents.length - 1; i++) { + if (childMapperName.isEmpty() == false) { + childMapperName.append("."); + } + childMapperName.append(pathComponents[i]); + + var childMapper = parent.mappers.get(childMapperName.toString()); + if (childMapper instanceof ObjectMapper objectMapper) { + current = objectMapper; + startPathComponent = i + 1; + pathInCurrent = pathInCurrent.substring(childMapperName.length() + 1); + break; + } + } + } + + return null; + } + protected SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers, boolean isFragment) { var fields = mappers.sorted(Comparator.comparing(Mapper::fullPath)) .map(Mapper::syntheticFieldLoader) @@ -828,10 +864,18 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { private final List fields; private final boolean isFragment; + private boolean storedFieldLoadersHaveValues; private boolean docValuesLoadersHaveValues; private boolean ignoredValuesPresent; private List ignoredValues; + // If this loader has anything to write. + // In special cases this can be false even if doc values loaders or stored field loaders + // have values. + // F.e. objects that only contain fields that are destinations of copy_to. + private boolean writersHaveValues; + // Use an ordered map between field names and writers to order writing by field name. + private TreeMap currentWriters; private SyntheticSourceFieldLoader(List fields, boolean isFragment) { this.fields = fields; @@ -882,9 +926,55 @@ public boolean advanceToDoc(int docId) throws IOException { } } + @Override + public void prepare() { + if ((storedFieldLoadersHaveValues || docValuesLoadersHaveValues || ignoredValuesPresent) == false) { + writersHaveValues = false; + return; + } + + for (var loader : fields) { + // Currently this logic is only relevant for object loaders. + if (loader instanceof ObjectMapper.SyntheticSourceFieldLoader objectSyntheticFieldLoader) { + objectSyntheticFieldLoader.prepare(); + } + } + + currentWriters = new TreeMap<>(); + + if (ignoredValues != null && ignoredValues.isEmpty() == false) { + for (IgnoredSourceFieldMapper.NameValue value : ignoredValues) { + if (value.hasValue()) { + writersHaveValues |= true; + } + + var existing = currentWriters.get(value.name()); + if (existing == null) { + currentWriters.put(value.name(), new FieldWriter.IgnoredSource(value)); + } else if (existing instanceof FieldWriter.IgnoredSource isw) { + isw.mergeWith(value); + } + } + } + + for (SourceLoader.SyntheticFieldLoader field : fields) { + if (field.hasValue()) { + if (currentWriters.containsKey(field.fieldName()) == false) { + writersHaveValues |= true; + currentWriters.put(field.fieldName(), new FieldWriter.FieldLoader(field)); + } else { + // Skip if the field source is stored separately, to avoid double-printing. + // Make sure to reset the state of loader so that values stored inside will not + // be used after this document is finished. + field.reset(); + } + } + } + } + @Override public boolean hasValue() { - return storedFieldLoadersHaveValues || docValuesLoadersHaveValues || ignoredValuesPresent; + return writersHaveValues; } @Override @@ -892,12 +982,13 @@ public void write(XContentBuilder b) throws IOException { if (hasValue() == false) { return; } + if (isRoot() && isEnabled() == false) { // If the root object mapper is disabled, it is expected to contain // the source encapsulated within a single ignored source value. assert ignoredValues.size() == 1 : ignoredValues.size(); XContentDataHelper.decodeAndWrite(b, ignoredValues.get(0).value()); - ignoredValues = null; + softReset(); return; } @@ -907,41 +998,12 @@ public void write(XContentBuilder b) throws IOException { b.startObject(leafName()); } - if (ignoredValues != null && ignoredValues.isEmpty() == false) { - // Use an ordered map between field names and writer functions, to order writing by field name. - Map orderedFields = new TreeMap<>(); - for (IgnoredSourceFieldMapper.NameValue value : ignoredValues) { - var existing = orderedFields.get(value.name()); - if (existing == null) { - orderedFields.put(value.name(), new FieldWriter.IgnoredSource(value)); - } else if (existing instanceof FieldWriter.IgnoredSource isw) { - isw.mergeWith(value); - } - } - for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field.hasValue()) { - if (orderedFields.containsKey(field.fieldName()) == false) { - orderedFields.put(field.fieldName(), new FieldWriter.FieldLoader(field)); - } else { - // Skip if the field source is stored separately, to avoid double-printing. - // Make sure to reset the state of loader so that values stored inside will not - // be used after this document is finished. - field.reset(); - } - } - } - - for (var writer : orderedFields.values()) { + for (var writer : currentWriters.values()) { + if (writer.hasValue()) { writer.writeTo(b); } - ignoredValues = null; - } else { - for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field.hasValue()) { - field.write(b); - } - } } + b.endObject(); softReset(); } @@ -957,6 +1019,8 @@ private void softReset() { storedFieldLoadersHaveValues = false; docValuesLoadersHaveValues = false; ignoredValuesPresent = false; + ignoredValues = null; + writersHaveValues = false; } @Override @@ -986,34 +1050,49 @@ public String fieldName() { interface FieldWriter { void writeTo(XContentBuilder builder) throws IOException; + boolean hasValue(); + record FieldLoader(SourceLoader.SyntheticFieldLoader loader) implements FieldWriter { @Override public void writeTo(XContentBuilder builder) throws IOException { loader.write(builder); } + + @Override + public boolean hasValue() { + return loader.hasValue(); + } } class IgnoredSource implements FieldWriter { private final String fieldName; private final String leafName; - private final List values; + private final List encodedValues; IgnoredSource(IgnoredSourceFieldMapper.NameValue initialValue) { this.fieldName = initialValue.name(); this.leafName = initialValue.getFieldName(); - this.values = new ArrayList<>(); - this.values.add(initialValue.value()); + this.encodedValues = new ArrayList<>(); + if (initialValue.hasValue()) { + this.encodedValues.add(initialValue.value()); + } } @Override public void writeTo(XContentBuilder builder) throws IOException { - XContentDataHelper.writeMerged(builder, leafName, values); + XContentDataHelper.writeMerged(builder, leafName, encodedValues); + } + + @Override + public boolean hasValue() { + return encodedValues.isEmpty() == false; } public FieldWriter mergeWith(IgnoredSourceFieldMapper.NameValue nameValue) { assert Objects.equals(nameValue.name(), fieldName) : "IgnoredSource is merged with wrong field data"; - - values.add(nameValue.value()); + if (nameValue.hasValue()) { + encodedValues.add(nameValue.value()); + } return this; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 3318595ed7129..118cdbffc5db9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -48,6 +48,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { "mapper.source.synthetic_source_with_copy_to_and_doc_values_false" ); public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_FIX = new NodeFeature("mapper.source.synthetic_source_copy_to_fix"); + public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature( + "mapper.source.synthetic_source_copy_to_inside_objects_fix" + ); public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index baff3835d104b..ec255a53e7c5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -209,6 +209,9 @@ public void write(LeafStoredFieldLoader storedFieldLoader, int docId, XContentBu if (docValuesLoader != null) { docValuesLoader.advanceToDoc(docId); } + + loader.prepare(); + // TODO accept a requested xcontent type if (loader.hasValue()) { loader.write(b); @@ -299,6 +302,16 @@ public String fieldName() { */ DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException; + /** + Perform any preprocessing needed before producing synthetic source + and deduce whether this mapper (and its children, if any) have values to write. + The expectation is for this method to be called before {@link SyntheticFieldLoader#hasValue()} + and {@link SyntheticFieldLoader#write(XContentBuilder)} are used. + */ + default void prepare() { + // Noop + } + /** * Has this field loaded any values for this document? */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 6ba0d4568e713..a6b2ad265decf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -42,12 +42,13 @@ import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; +import java.util.SortedMap; +import java.util.TreeMap; /** * Mapper for {@code _tsid} field included generated when the index is @@ -175,16 +176,14 @@ public static class TimeSeriesIdBuilder implements DocumentDimensions { public static final int MAX_DIMENSIONS = 512; - private record Dimension(BytesRef name, BytesReference value) {} - private final Murmur3Hasher tsidHasher = new Murmur3Hasher(0); /** - * A sorted set of the serialized values of dimension fields that will be used + * A map of the serialized values of dimension fields that will be used * for generating the _tsid field. The map will be used by {@link TimeSeriesIdFieldMapper} * to build the _tsid field for the document. */ - private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); + private final SortedMap> dimensions = new TreeMap<>(); /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ @@ -202,9 +201,17 @@ public BytesReference buildLegacyTsid() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); - for (Dimension entry : dimensions) { - out.writeBytesRef(entry.name); - entry.value.writeTo(out); + for (Map.Entry> entry : dimensions.entrySet()) { + out.writeBytesRef(entry.getKey()); + List value = entry.getValue(); + if (value.size() > 1) { + // multi-value dimensions are only supported for newer indices that use buildTsidHash + throw new IllegalArgumentException( + "Dimension field [" + entry.getKey().utf8ToString() + "] cannot be a multi-valued field." + ); + } + assert value.isEmpty() == false : "dimension value is empty"; + value.get(0).writeTo(out); } return out.bytes(); } @@ -236,18 +243,19 @@ public BytesReference buildTsidHash() { int tsidHashIndex = StreamOutput.putVInt(tsidHash, len, 0); tsidHasher.reset(); - for (final Dimension dimension : dimensions) { - tsidHasher.update(dimension.name.bytes); + for (final BytesRef name : dimensions.keySet()) { + tsidHasher.update(name.bytes); } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); // NOTE: concatenate all dimension value hashes up to a certain number of dimensions int tsidHashStartIndex = tsidHashIndex; - for (final Dimension dimension : dimensions) { + for (final List values : dimensions.values()) { if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) { break; } - final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); + assert values.isEmpty() == false : "dimension values are empty"; + final BytesRef dimensionValueBytesRef = values.get(0).toBytesRef(); ByteUtils.writeIntLE( StringHelper.murmurhash3_x86_32( dimensionValueBytesRef.bytes, @@ -263,8 +271,10 @@ public BytesReference buildTsidHash() { // NOTE: hash all dimension field allValues tsidHasher.reset(); - for (final Dimension dimension : dimensions) { - tsidHasher.update(dimension.value.toBytesRef().bytes); + for (final List values : dimensions.values()) { + for (BytesReference v : values) { + tsidHasher.update(v.toBytesRef().bytes); + } } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); @@ -367,8 +377,20 @@ public DocumentDimensions validate(final IndexSettings settings) { } private void add(String fieldName, BytesReference encoded) throws IOException { - if (dimensions.add(new Dimension(new BytesRef(fieldName), encoded)) == false) { - throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); + BytesRef name = new BytesRef(fieldName); + List values = dimensions.get(name); + if (values == null) { + // optimize for the common case where dimensions are not multi-valued + dimensions.put(name, List.of(encoded)); + } else { + if (values.size() == 1) { + // converts the immutable list that's optimized for the common case of having only one value to a mutable list + BytesReference previousValue = values.get(0); + values = new ArrayList<>(4); + values.add(previousValue); + dimensions.put(name, values); + } + values.add(encoded); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index 354f0ec92b0fa..8bacaf8505f91 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -72,7 +72,7 @@ public static BytesRef encodeXContentBuilder(XContentBuilder builder) throws IOE } /** - * Returns a special encoded value that signals that values of this field + * Returns a special encoded value that signals that this field * should not be present in synthetic source. * * An example is a field that has values copied to it using copy_to. @@ -80,7 +80,7 @@ public static BytesRef encodeXContentBuilder(XContentBuilder builder) throws IOE * synthetic _source same as it wouldn't be present in stored source. * @return */ - public static BytesRef nothing() { + public static BytesRef voidValue() { return new BytesRef(new byte[] { VOID_ENCODING }); } @@ -112,41 +112,27 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { /** * Writes encoded values to provided builder. If there are multiple values they are merged into * a single resulting array. + * + * Note that this method assumes all encoded parts have values that need to be written (are not VOID encoded). * @param b destination * @param fieldName name of the field that is written * @param encodedParts subset of field data encoded using methods of this class. Can contain arrays which will be flattened. * @throws IOException */ static void writeMerged(XContentBuilder b, String fieldName, List encodedParts) throws IOException { - var partsWithData = 0; - for (BytesRef encodedPart : encodedParts) { - if (isDataPresent(encodedPart)) { - partsWithData++; - } - } - - if (partsWithData == 0) { + if (encodedParts.isEmpty()) { return; } - if (partsWithData == 1) { + if (encodedParts.size() == 1) { b.field(fieldName); - for (BytesRef encodedPart : encodedParts) { - if (isDataPresent(encodedPart)) { - XContentDataHelper.decodeAndWrite(b, encodedPart); - } - } - + XContentDataHelper.decodeAndWrite(b, encodedParts.get(0)); return; } b.startArray(fieldName); for (var encodedValue : encodedParts) { - if (isDataPresent(encodedValue) == false) { - continue; - } - Optional encodedXContentType = switch ((char) encodedValue.bytes[encodedValue.offset]) { case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of( getXContentType(encodedValue) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java index 3f9ba85bee1bb..23d2c4b554d85 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java @@ -76,8 +76,4 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { } } - @Override - public void close() { - // no-op - } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index c4f0d72c06a80..62d2aa1f026f7 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1514,6 +1514,22 @@ public void forceMerge(ForceMergeRequest forceMerge) throws IOException { engine.forceMerge(forceMerge.flush(), forceMerge.maxNumSegments(), forceMerge.onlyExpungeDeletes(), forceMerge.forceMergeUUID()); } + public void triggerPendingMerges() throws IOException { + switch (state /* single volatile read */) { + case STARTED, POST_RECOVERY -> getEngine().forceMerge( + // don't immediately flush - if any merging happens then we don't wait for it anyway + false, + // don't apply any segment count limit, we only want to call IndexWriter#maybeMerge + ForceMergeRequest.Defaults.MAX_NUM_SEGMENTS, + // don't look for expunge-delete merges, we only want to call IndexWriter#maybeMerge + false, + // force-merge UUID is not used when calling IndexWriter#maybeMerge + null + ); + // otherwise shard likely closed and maybe reopened, nothing to do + } + } + /** * Creates a new {@link IndexCommit} snapshot from the currently running engine. All resources referenced by this * commit won't be freed until the commit / snapshot is closed. diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 89f651468068d..43bfe7e0e7e2c 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -262,6 +262,7 @@ public class IndicesService extends AbstractLifecycleComponent private final TimestampFieldMapperService timestampFieldMapperService; private final CheckedBiConsumer requestCacheKeyDifferentiator; private final MapperMetrics mapperMetrics; + private final PostRecoveryMerger postRecoveryMerger; @Override protected void doStart() { @@ -378,6 +379,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon clusterService.getClusterSettings().addSettingsUpdateConsumer(ALLOW_EXPENSIVE_QUERIES, this::setAllowExpensiveQueries); this.timestampFieldMapperService = new TimestampFieldMapperService(settings, threadPool, this); + + this.postRecoveryMerger = new PostRecoveryMerger(settings, threadPool.executor(ThreadPool.Names.FORCE_MERGE), this::getShardOrNull); } private static final String DANGLING_INDICES_UPDATE_THREAD_NAME = "DanglingIndices#updateTask"; @@ -890,23 +893,29 @@ public void createShard( RecoveryState recoveryState = indexService.createRecoveryState(shardRouting, targetNode, sourceNode); IndexShard indexShard = indexService.createShard(shardRouting, globalCheckpointSyncer, retentionLeaseSyncer); indexShard.addShardFailureCallback(onShardFailure); - indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService, (mapping, listener) -> { - assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS - : "mapping update consumer only required by local shards recovery"; - AcknowledgedRequest putMappingRequestAcknowledgedRequest = new PutMappingRequest().setConcreteIndex( - shardRouting.index() - ) - .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid - .source(mapping.source().string(), XContentType.JSON); - // concrete index - no name clash, it uses uuid - client.execute( - featureService.clusterHasFeature(clusterService.state(), SUPPORTS_AUTO_PUT) - ? TransportAutoPutMappingAction.TYPE - : TransportPutMappingAction.TYPE, - putMappingRequestAcknowledgedRequest.ackTimeout(TimeValue.MAX_VALUE).masterNodeTimeout(TimeValue.MAX_VALUE), - new RefCountAwareThreadedActionListener<>(threadPool.generic(), listener.map(ignored -> null)) - ); - }, this, clusterStateVersion); + indexShard.startRecovery( + recoveryState, + recoveryTargetService, + postRecoveryMerger.maybeMergeAfterRecovery(shardRouting, recoveryListener), + repositoriesService, + (mapping, listener) -> { + assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS + : "mapping update consumer only required by local shards recovery"; + AcknowledgedRequest putMappingRequestAcknowledgedRequest = new PutMappingRequest() + // concrete index - no name clash, it uses uuid + .setConcreteIndex(shardRouting.index()) + .source(mapping.source().string(), XContentType.JSON); + client.execute( + featureService.clusterHasFeature(clusterService.state(), SUPPORTS_AUTO_PUT) + ? TransportAutoPutMappingAction.TYPE + : TransportPutMappingAction.TYPE, + putMappingRequestAcknowledgedRequest.ackTimeout(TimeValue.MAX_VALUE).masterNodeTimeout(TimeValue.MAX_VALUE), + new RefCountAwareThreadedActionListener<>(threadPool.generic(), listener.map(ignored -> null)) + ); + }, + this, + clusterStateVersion + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java b/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java new file mode 100644 index 0000000000000..11c8a2b4d6048 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.indices; + +import org.apache.lucene.index.IndexWriter; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Strings; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardLongFieldRange; +import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; +import org.elasticsearch.indices.recovery.RecoveryFailedException; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.concurrent.Executor; +import java.util.function.Function; + +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_HOT_NODE_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.INDEX_ROLE; + +/** + * Triggers a check for pending merges when a shard completes recovery. + */ +class PostRecoveryMerger { + + private static final Logger logger = LogManager.getLogger(PostRecoveryMerger.class); + + private static final boolean TRIGGER_MERGE_AFTER_RECOVERY; + + static { + final var propertyValue = System.getProperty("es.trigger_merge_after_recovery"); + if (propertyValue == null) { + TRIGGER_MERGE_AFTER_RECOVERY = true; + } else if ("false".equals(propertyValue)) { + TRIGGER_MERGE_AFTER_RECOVERY = false; + } else { + throw new IllegalStateException( + "system property [es.trigger_merge_after_recovery] may only be set to [false], but was [" + propertyValue + "]" + ); + } + } + + /** + * Throttled runner to avoid multiple concurrent calls to {@link IndexWriter#maybeMerge()}: we do not need to execute these things + * especially quickly, as long as they happen eventually, and each such call may involve some IO (reading the soft-deletes doc values to + * count deleted docs). Note that we're not throttling any actual merges, just the checks to see what merges might be needed. Throttling + * merges across shards is a separate issue, but normally this mechanism won't trigger any new merges anyway. + */ + private final ThrottledTaskRunner postRecoveryMergeRunner; + + private final Function shardFunction; + private final boolean enabled; + + PostRecoveryMerger(Settings settings, Executor executor, Function shardFunction) { + this.postRecoveryMergeRunner = new ThrottledTaskRunner(getClass().getCanonicalName(), 1, executor); + this.shardFunction = shardFunction; + this.enabled = + // enabled globally ... + TRIGGER_MERGE_AFTER_RECOVERY + // ... and we are a node that expects nontrivial amounts of indexing work + && (DiscoveryNode.hasRole(settings, DATA_HOT_NODE_ROLE) + || DiscoveryNode.hasRole(settings, DATA_CONTENT_NODE_ROLE) + || DiscoveryNode.hasRole(settings, DATA_ROLE) + || DiscoveryNode.hasRole(settings, INDEX_ROLE)); + } + + PeerRecoveryTargetService.RecoveryListener maybeMergeAfterRecovery( + ShardRouting shardRouting, + PeerRecoveryTargetService.RecoveryListener recoveryListener + ) { + if (enabled == false) { + return recoveryListener; + } + + if (shardRouting.isPromotableToPrimary() == false) { + return recoveryListener; + } + + final var shardId = shardRouting.shardId(); + return new PeerRecoveryTargetService.RecoveryListener() { + @Override + public void onRecoveryDone( + RecoveryState state, + ShardLongFieldRange timestampMillisFieldRange, + ShardLongFieldRange eventIngestedMillisFieldRange + ) { + postRecoveryMergeRunner.enqueueTask(new PostRecoveryMerge(shardId)); + recoveryListener.onRecoveryDone(state, timestampMillisFieldRange, eventIngestedMillisFieldRange); + } + + @Override + public void onRecoveryFailure(RecoveryFailedException e, boolean sendShardFailure) { + recoveryListener.onRecoveryFailure(e, sendShardFailure); + } + }; + } + + class PostRecoveryMerge implements ActionListener { + private final ShardId shardId; + + PostRecoveryMerge(ShardId shardId) { + this.shardId = shardId; + } + + @Override + public void onResponse(Releasable releasable) { + try (releasable) { + final var indexShard = shardFunction.apply(shardId); + if (indexShard == null) { + return; + } + + indexShard.triggerPendingMerges(); + } catch (Exception e) { + logFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + logFailure(e); + } + + private void logFailure(Exception e) { + // post-recovery merge is a best-effort thing, failure needs no special handling + logger.debug(() -> Strings.format("failed to execute post-recovery merge of [%s]", shardId), e); + } + } +} diff --git a/libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java b/server/src/main/java/org/elasticsearch/plugins/ExtendedPluginsClassLoader.java similarity index 94% rename from libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java rename to server/src/main/java/org/elasticsearch/plugins/ExtendedPluginsClassLoader.java index be3e76bd83396..d9bf0d653bb62 100644 --- a/libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java +++ b/server/src/main/java/org/elasticsearch/plugins/ExtendedPluginsClassLoader.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.plugins.loader; +package org.elasticsearch.plugins; import java.security.AccessController; import java.security.PrivilegedAction; @@ -17,7 +17,7 @@ /** * A classloader that is a union over the parent core classloader and classloaders of extended plugins. */ -public class ExtendedPluginsClassLoader extends ClassLoader { +class ExtendedPluginsClassLoader extends ClassLoader { /** Loaders of plugins extended by a plugin. */ private final List extendedLoaders; diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java b/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java deleted file mode 100644 index a5ca26283231d..0000000000000 --- a/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.plugins; - -import org.elasticsearch.plugins.loader.ExtendedPluginsClassLoader; - -import java.util.List; - -// TODO: remove this indirection now that transport client is gone -/** - * This class exists solely as an intermediate layer to avoid causing PluginsService - * to load ExtendedPluginsClassLoader when used in the transport client. - */ -class PluginLoaderIndirection { - - static ClassLoader createLoader(ClassLoader parent, List extendedLoaders) { - return ExtendedPluginsClassLoader.create(parent, extendedLoaders); - } -} diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 47f0a50ff309c..d5dd6d62d615e 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -468,7 +468,7 @@ private void loadBundle( ); } - final ClassLoader parentLoader = PluginLoaderIndirection.createLoader( + final ClassLoader parentLoader = ExtendedPluginsClassLoader.create( getClass().getClassLoader(), extendedPlugins.stream().map(LoadedPlugin::loader).toList() ); diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index e48677f46d57a..17eda305b5ccf 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -105,19 +105,19 @@ public boolean isContentConsumed() { protected RestRequest( XContentParserConfiguration parserConfig, Map params, - String path, + String rawPath, Map> headers, HttpRequest httpRequest, HttpChannel httpChannel ) { - this(parserConfig, params, path, headers, httpRequest, httpChannel, requestIdGenerator.incrementAndGet()); + this(parserConfig, params, rawPath, headers, httpRequest, httpChannel, requestIdGenerator.incrementAndGet()); } @SuppressWarnings("this-escape") private RestRequest( XContentParserConfiguration parserConfig, Map params, - String path, + String rawPath, Map> headers, HttpRequest httpRequest, HttpChannel httpChannel, @@ -149,7 +149,7 @@ private RestRequest( : parserConfig.withRestApiVersion(effectiveApiVersion); this.httpChannel = httpChannel; this.params = params; - this.rawPath = path; + this.rawPath = rawPath; this.headers = Collections.unmodifiableMap(headers); this.requestId = requestId; } @@ -204,11 +204,10 @@ void ensureSafeBuffers() { */ public static RestRequest request(XContentParserConfiguration parserConfig, HttpRequest httpRequest, HttpChannel httpChannel) { Map params = params(httpRequest.uri()); - String path = path(httpRequest.uri()); return new RestRequest( parserConfig, params, - path, + httpRequest.rawPath(), httpRequest.getHeaders(), httpRequest, httpChannel, @@ -229,15 +228,6 @@ private static Map params(final String uri) { return params; } - private static String path(final String uri) { - final int index = uri.indexOf('?'); - if (index >= 0) { - return uri.substring(0, index); - } else { - return uri; - } - } - /** * Creates a new REST request. The path is not decoded so this constructor will not throw a * {@link BadParameterException}. diff --git a/server/src/main/java/org/elasticsearch/search/MultiValueMode.java b/server/src/main/java/org/elasticsearch/search/MultiValueMode.java index 7f499da9c0fc8..49480816bbbb1 100644 --- a/server/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/server/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -593,12 +593,7 @@ public boolean advanceExact(int parentDoc) throws IOException { return true; } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); lastSeenParentDoc = parentDoc; lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren); @@ -700,12 +695,7 @@ public boolean advanceExact(int parentDoc) throws IOException { return true; } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); lastSeenParentDoc = parentDoc; lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren); @@ -713,12 +703,22 @@ public boolean advanceExact(int parentDoc) throws IOException { } @Override - public double doubleValue() throws IOException { + public double doubleValue() { return lastEmittedValue; } }; } + private static int getFirstChildDoc(int prevParentDoc, DocIdSetIterator childDocs) throws IOException { + final int firstChildDoc; + if (childDocs.docID() > prevParentDoc) { + firstChildDoc = childDocs.docID(); + } else { + firstChildDoc = childDocs.advance(prevParentDoc + 1); + } + return firstChildDoc; + } + protected double pick( SortedNumericDoubleValues values, double missingValue, @@ -754,7 +754,7 @@ public boolean advanceExact(int target) throws IOException { } @Override - public BytesRef binaryValue() throws IOException { + public BytesRef binaryValue() { return this.value; } }; @@ -768,14 +768,13 @@ public boolean advanceExact(int target) throws IOException { if (values.advanceExact(target)) { value = pick(values); return true; - } else { - value = missingValue; - return missingValue != null; } + value = missingValue; + return missingValue != null; } @Override - public BytesRef binaryValue() throws IOException { + public BytesRef binaryValue() { return value; } }; @@ -825,12 +824,7 @@ public boolean advanceExact(int parentDoc) throws IOException { } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); lastSeenParentDoc = parentDoc; lastEmittedValue = pick(selectedValues, builder, childDocs, firstChildDoc, parentDoc, maxChildren); @@ -841,7 +835,7 @@ public boolean advanceExact(int parentDoc) throws IOException { } @Override - public BytesRef binaryValue() throws IOException { + public BytesRef binaryValue() { return lastEmittedValue; } }; @@ -964,12 +958,7 @@ public boolean advanceExact(int parentDoc) throws IOException { } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); docID = lastSeenParentDoc = parentDoc; lastEmittedOrd = pick(selectedValues, childDocs, firstChildDoc, parentDoc, maxChildren); diff --git a/server/src/main/resources/META-INF/services/java.util.spi.CalendarDataProvider b/server/src/main/resources/META-INF/services/java.util.spi.CalendarDataProvider deleted file mode 100644 index 6e006ccf815f8..0000000000000 --- a/server/src/main/resources/META-INF/services/java.util.spi.CalendarDataProvider +++ /dev/null @@ -1 +0,0 @@ -org.elasticsearch.common.time.IsoCalendarDataProvider \ No newline at end of file diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 756e8106f631a..55abdc84fc8fb 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -53,11 +53,6 @@ grant codeBase "${codebase.lucene-misc}" { permission java.nio.file.LinkPermission "hard"; }; -grant codeBase "${codebase.elasticsearch-plugin-classloader}" { - // needed to create the classloader which allows plugins to extend other plugins - permission java.lang.RuntimePermission "createClassLoader"; -}; - grant codeBase "${codebase.elasticsearch-core}" { permission java.lang.RuntimePermission "createClassLoader"; permission java.lang.RuntimePermission "getClassLoader"; diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java index f7fc248376e47..e3ab7463ad941 100644 --- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java +++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -199,4 +200,40 @@ public void testToString() { assertEquals("2000099", TransportVersion.fromId(2_00_00_99).toString()); assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString()); } + + /** + * Until 9.0 bumps its transport version to 9_000_00_0, all transport changes must be backported to 8.x. + * This test ensures transport versions are dense, so that we have confidence backports have not been missed. + * Note that it does not ensure patches are not missed, but it should catch the majority of misordered + * or missing transport versions. + */ + public void testDenseTransportVersions() { + Set missingVersions = new TreeSet<>(); + TransportVersion previous = null; + for (var tv : TransportVersions.getAllVersions()) { + if (tv.before(TransportVersions.V_8_14_0)) { + continue; + } + if (previous == null) { + previous = tv; + continue; + } + + if (previous.id() + 1000 < tv.id()) { + int nextId = previous.id(); + do { + nextId = (nextId + 1000) / 1000 * 1000; + missingVersions.add(nextId); + } while (nextId + 1000 < tv.id()); + } + previous = tv; + } + if (missingVersions.isEmpty() == false) { + StringBuilder msg = new StringBuilder("Missing transport versions:\n"); + for (Integer id : missingVersions) { + msg.append(" " + id + "\n"); + } + fail(msg.toString()); + } + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java index 68a161d426f48..82463d601d164 100644 --- a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java @@ -536,7 +536,7 @@ public void sendExecuteRankFeature( // override the RankFeaturePhase to raise an exception RankFeaturePhase rankFeaturePhase = new RankFeaturePhase(results, null, mockSearchPhaseContext, null) { @Override - void innerRun() { + void innerRun(RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext) { throw new IllegalArgumentException("simulated failure"); } @@ -1142,7 +1142,13 @@ public void moveToNextPhase( ) { // this is called after the RankFeaturePhaseCoordinatorContext has been executed phaseDone.set(true); - finalResults[0] = reducedQueryPhase.sortedTopDocs().scoreDocs(); + try { + finalResults[0] = reducedQueryPhase == null + ? queryPhaseResults.reduce().sortedTopDocs().scoreDocs() + : reducedQueryPhase.sortedTopDocs().scoreDocs(); + } catch (Exception e) { + throw new AssertionError(e); + } logger.debug("Skipping moving to next phase"); } }; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 81c19946753db..bbcf1ca33a0c2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -244,8 +244,8 @@ public void testCreateDataStreamWithFailureStoreInitialized() throws Exception { ActionListener.noop(), true ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); @@ -284,8 +284,8 @@ public void testCreateDataStreamWithFailureStoreUninitialized() throws Exception ActionListener.noop(), false ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); @@ -321,8 +321,8 @@ public void testCreateDataStreamWithFailureStoreWithRefreshRate() throws Excepti ActionListener.noop(), true ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().index(backingIndexName), notNullValue()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index b0b229f19dd7d..0354b6f0bcea8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.SnapshotsInProgress; @@ -132,8 +131,10 @@ public void testDeleteUnassigned() throws Exception { before, service.executor, List.of( - new DeleteIndexClusterStateUpdateRequest(ActionListener.noop()).indices( - new Index[] { before.metadata().getIndices().get(index).getIndex() } + new MetadataDeleteIndexService.DeleteIndicesClusterStateUpdateTask( + Set.of(before.metadata().getIndices().get(index).getIndex()), + TEST_REQUEST_TIMEOUT, + ActionListener.noop() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java index 796d0892184f5..4f2c84d76b5a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java @@ -693,10 +693,14 @@ public void testAddAndRemoveAliasClusterStateUpdate() throws Exception { String index = randomAlphaOfLength(5); ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index); IndicesAliasesClusterStateUpdateRequest addAliasRequest = new IndicesAliasesClusterStateUpdateRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, List.of(new AliasAction.Add(index, "test", null, null, null, null, null)), List.of(AliasActionResult.buildSuccess(List.of(index), AliasActions.add().aliases("test").indices(index))) ); IndicesAliasesClusterStateUpdateRequest removeAliasRequest = new IndicesAliasesClusterStateUpdateRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, List.of(new AliasAction.Remove(index, "test", true)), List.of(AliasActionResult.buildSuccess(List.of(index), AliasActions.remove().aliases("test").indices(index))) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 6ac8e6a853f27..e39ccdf7af5e2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -595,7 +595,32 @@ public void testRoutingPathBooleansInSource() throws IOException { IndexRouting routing = indexRoutingForPath(shards, "foo"); assertIndexShard(routing, Map.of("foo", true), Math.floorMod(hash(List.of("foo", "true")), shards)); assertIndexShard(routing, Map.of("foo", false), Math.floorMod(hash(List.of("foo", "false")), shards)); + } + public void testRoutingPathArraysInSource() throws IOException { + int shards = between(2, 1000); + IndexRouting routing = indexRoutingForPath(shards, "a,b,c,d"); + assertIndexShard( + routing, + Map.of("c", List.of(true), "d", List.of(), "a", List.of("foo", "bar", "foo"), "b", List.of(21, 42)), + // Note that the fields are sorted + Math.floorMod(hash(List.of("a", "foo", "a", "bar", "a", "foo", "b", "21", "b", "42", "c", "true")), shards) + ); + } + + public void testRoutingPathObjectArraysInSource() throws IOException { + int shards = between(2, 1000); + IndexRouting routing = indexRoutingForPath(shards, "a"); + + BytesReference source = source(Map.of("a", List.of("foo", Map.of("foo", "bar")))); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source, s -> {}) + ); + assertThat( + e.getMessage(), + equalTo("Error extracting routing: Failed to parse object: expecting value token but found [START_OBJECT]") + ); } public void testRoutingPathBwc() throws IOException { @@ -668,7 +693,11 @@ private void assertIndexShard(IndexRouting routing, Map source, IndexRouting.ExtractFromSource.Builder b = r.builder(); for (Map.Entry e : flattened.entrySet()) { - b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); + if (e.getValue() instanceof List listValue) { + listValue.forEach(v -> b.addMatching(e.getKey(), new BytesRef(v.toString()))); + } else { + b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); + } } String idFromBuilder = b.createId(suffix, () -> { throw new AssertionError(); }); assertThat(idFromBuilder, equalTo(idFromSource)); diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index 78e900898bfc9..498c04c005304 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; @@ -27,7 +28,6 @@ import org.elasticsearch.cluster.LocalMasterServiceTask; import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.SimpleBatchedExecutor; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.coordination.ClusterStatePublisher; import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; @@ -1571,7 +1571,7 @@ public void onAckFailure(Exception e) { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(TimeValue.ZERO, null), null) { + new AckedClusterStateUpdateTask(ackedRequest(TimeValue.ZERO, MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT), null) { @Override public ClusterState execute(ClusterState currentState) { return ClusterState.builder(currentState).build(); @@ -1623,7 +1623,7 @@ public void onAckTimeout() { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(ackTimeout, null), null) { + new AckedClusterStateUpdateTask(ackedRequest(ackTimeout, MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT), null) { @Override public ClusterState execute(ClusterState currentState) { threadPool.getThreadContext().addResponseHeader(responseHeaderName, responseHeaderValue); @@ -1678,7 +1678,10 @@ public void onAckTimeout() { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT, null), null) { + new AckedClusterStateUpdateTask( + ackedRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT, MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT), + null + ) { @Override public ClusterState execute(ClusterState currentState) { return ClusterState.builder(currentState).build(); @@ -2657,20 +2660,15 @@ public static ClusterState discoveryState(MasterService masterService) { } /** - * Returns a plain {@link AckedRequest} that does not implement any functionality outside of the timeout getters. + * Returns a plain {@link AcknowledgedRequest} that does not implement any functionality outside of the timeout getters. */ - public static AckedRequest ackedRequest(TimeValue ackTimeout, TimeValue masterNodeTimeout) { - return new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return ackTimeout; - } - - @Override - public TimeValue masterNodeTimeout() { - return masterNodeTimeout; + public static AcknowledgedRequest ackedRequest(TimeValue ackTimeout, TimeValue masterNodeTimeout) { + class BareAcknowledgedRequest extends AcknowledgedRequest { + BareAcknowledgedRequest() { + super(masterNodeTimeout, ackTimeout); } - }; + } + return new BareAcknowledgedRequest(); } /** diff --git a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java index 76add3fea71c3..dfe3cf10fd494 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java @@ -25,6 +25,7 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; +import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAccessor; import java.util.List; import java.util.Locale; @@ -85,38 +86,32 @@ private void assertDateMathEquals(String text, String expected, String pattern) } private void assertDateMathEquals(String text, String expected, String pattern, Locale locale) { - long gotMillisJava = dateMathToMillis(text, DateFormatter.forPattern(pattern), locale); - long expectedMillis = DateFormatters.from(DateFormatter.forPattern("strict_date_optional_time").withLocale(locale).parse(expected)) - .toInstant() - .toEpochMilli(); + Instant gotInstant = dateMathToInstant(text, DateFormatter.forPattern(pattern), locale).truncatedTo(ChronoUnit.MILLIS); + Instant expectedInstant = DateFormatters.from( + DateFormatter.forPattern("strict_date_optional_time").withLocale(locale).parse(expected) + ).toInstant().truncatedTo(ChronoUnit.MILLIS); - assertThat(gotMillisJava, equalTo(expectedMillis)); + assertThat(gotInstant, equalTo(expectedInstant)); } public void testWeekBasedDates() { - // as per WeekFields.ISO first week starts on Monday and has minimum 4 days + // the years and weeks this outputs depends on where the first day of the first week is for each year DateFormatter dateFormatter = DateFormatters.forPattern("YYYY-ww"); - // first week of 2016 starts on Monday 2016-01-04 as previous week in 2016 has only 3 days assertThat( - DateFormatters.from(dateFormatter.parse("2016-01")), - equalTo(ZonedDateTime.of(2016, 01, 04, 0, 0, 0, 0, ZoneOffset.UTC)) + DateFormatters.from(dateFormatter.parse("2016-02")), + equalTo(ZonedDateTime.of(2016, 01, 03, 0, 0, 0, 0, ZoneOffset.UTC)) ); - // first week of 2015 starts on Monday 2014-12-29 because 4days belong to 2019 assertThat( - DateFormatters.from(dateFormatter.parse("2015-01")), - equalTo(ZonedDateTime.of(2014, 12, 29, 0, 0, 0, 0, ZoneOffset.UTC)) + DateFormatters.from(dateFormatter.parse("2015-02")), + equalTo(ZonedDateTime.of(2015, 01, 04, 0, 0, 0, 0, ZoneOffset.UTC)) ); - // as per WeekFields.ISO first week starts on Monday and has minimum 4 days dateFormatter = DateFormatters.forPattern("YYYY"); - // first week of 2016 starts on Monday 2016-01-04 as previous week in 2016 has only 3 days - assertThat(DateFormatters.from(dateFormatter.parse("2016")), equalTo(ZonedDateTime.of(2016, 01, 04, 0, 0, 0, 0, ZoneOffset.UTC))); - - // first week of 2015 starts on Monday 2014-12-29 because 4days belong to 2019 - assertThat(DateFormatters.from(dateFormatter.parse("2015")), equalTo(ZonedDateTime.of(2014, 12, 29, 0, 0, 0, 0, ZoneOffset.UTC))); + assertThat(DateFormatters.from(dateFormatter.parse("2016")), equalTo(ZonedDateTime.of(2015, 12, 27, 0, 0, 0, 0, ZoneOffset.UTC))); + assertThat(DateFormatters.from(dateFormatter.parse("2015")), equalTo(ZonedDateTime.of(2014, 12, 28, 0, 0, 0, 0, ZoneOffset.UTC))); } public void testEpochMillisParser() { @@ -600,8 +595,8 @@ public void testYearWithoutMonthRoundUp() { assertDateMathEquals("1500", "1500-01-01T23:59:59.999", "uuuu"); assertDateMathEquals("2022", "2022-01-01T23:59:59.999", "uuuu"); assertDateMathEquals("2022", "2022-01-01T23:59:59.999", "yyyy"); - // cannot reliably default week based years due to locale changing. See JavaDateFormatter javadocs - assertDateMathEquals("2022", "2022-01-03T23:59:59.999", "YYYY", Locale.ROOT); + // weird locales can change this to epoch-based + assertDateMathEquals("2022", "2021-12-26T23:59:59.999", "YYYY", Locale.ROOT); } private void assertRoundupFormatter(String format, String input, long expectedMilliSeconds) { @@ -789,30 +784,28 @@ public void testExceptionWhenCompositeParsingFailsDateMath() { String text = "2014-06-06T12:01:02.123"; ElasticsearchParseException e1 = expectThrows( ElasticsearchParseException.class, - () -> dateMathToMillis(text, DateFormatter.forPattern(pattern), randomLocale(random())) + () -> dateMathToInstant(text, DateFormatter.forPattern(pattern), randomLocale(random())) ); assertThat(e1.getMessage(), containsString(pattern)); assertThat(e1.getMessage(), containsString(text)); } - private long dateMathToMillis(String text, DateFormatter dateFormatter, Locale locale) { + private Instant dateMathToInstant(String text, DateFormatter dateFormatter, Locale locale) { DateFormatter javaFormatter = dateFormatter.withLocale(locale); DateMathParser javaDateMath = javaFormatter.toDateMathParser(); - return javaDateMath.parse(text, () -> 0, true, (ZoneId) null).toEpochMilli(); + return javaDateMath.parse(text, () -> 0, true, null); } public void testDayOfWeek() { - // 7 (ok joda) vs 1 (java by default) but 7 with customized org.elasticsearch.common.time.IsoLocale.ISO8601 ZonedDateTime now = LocalDateTime.of(2009, 11, 15, 1, 32, 8, 328402).atZone(ZoneOffset.UTC); // Sunday DateFormatter javaFormatter = DateFormatter.forPattern("8e").withZone(ZoneOffset.UTC); - assertThat(javaFormatter.format(now), equalTo("7")); + assertThat(javaFormatter.format(now), equalTo("1")); } public void testStartOfWeek() { - // 2019-21 (ok joda) vs 2019-22 (java by default) but 2019-21 with customized org.elasticsearch.common.time.IsoLocale.ISO8601 ZonedDateTime now = LocalDateTime.of(2019, 5, 26, 1, 32, 8, 328402).atZone(ZoneOffset.UTC); DateFormatter javaFormatter = DateFormatter.forPattern("8YYYY-ww").withZone(ZoneOffset.UTC); - assertThat(javaFormatter.format(now), equalTo("2019-21")); + assertThat(javaFormatter.format(now), equalTo("2019-22")); } // these parsers should allow both ',' and '.' as a decimal point diff --git a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java index 2e8394f920029..cd386d9159aa6 100644 --- a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java @@ -19,6 +19,7 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatterBuilder; import java.time.format.ResolverStyle; +import java.time.temporal.ChronoUnit; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongSupplier; @@ -82,12 +83,12 @@ public void testOverridingLocaleOrZoneAndCompositeRoundUpParser() { // the pattern has to be composite and the match should not be on the first one DateFormatter formatter = DateFormatter.forPattern("date||epoch_millis").withLocale(randomLocale(random())); DateMathParser parser = formatter.toDateMathParser(); - long gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli(); + Instant gotMillis = parser.parse("297276785531", () -> 0, true, null).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(gotMillis, "297276785531", "297276785531"); formatter = DateFormatter.forPattern("date||epoch_millis").withZone(ZoneOffset.UTC); parser = formatter.toDateMathParser(); - gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli(); + gotMillis = parser.parse("297276785531", () -> 0, true, null).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(gotMillis, "297276785531", "297276785531"); } @@ -99,7 +100,7 @@ public void testWeekBasedDate() { // defaulting missing day of week formatter = DateFormatter.forPattern("YYYY'W'ww[e]");// YYYY'W'wwe // second week of 2022 is starting on Monday 10th Jan - assertDateMathEquals(formatter.toDateMathParser(), "2022W02", "2022-01-10T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2022W02", "2022-01-02T23:59:59.999Z", 0, true, ZoneOffset.UTC); } public void testDayOfYear() { @@ -125,16 +126,16 @@ public void testAMPMWithTimeMissing() { public void testWeekDates() { DateFormatter formatter = DateFormatter.forPattern("YYYY-ww"); - assertDateMathEquals(formatter.toDateMathParser(), "2016-01", "2016-01-04T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2016-02", "2016-01-03T23:59:59.999Z", 0, true, ZoneOffset.UTC); formatter = DateFormatter.forPattern("YYYY"); - assertDateMathEquals(formatter.toDateMathParser(), "2016", "2016-01-04T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2016", "2015-12-27T23:59:59.999Z", 0, true, ZoneOffset.UTC); formatter = DateFormatter.forPattern("YYYY-ww"); - assertDateMathEquals(formatter.toDateMathParser(), "2015-01", "2014-12-29T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2015-02", "2015-01-04T23:59:59.999Z", 0, true, ZoneOffset.UTC); formatter = DateFormatter.forPattern("YYYY"); - assertDateMathEquals(formatter.toDateMathParser(), "2015", "2014-12-29T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2015", "2014-12-28T23:59:59.999Z", 0, true, ZoneOffset.UTC); } public void testBasicDates() { @@ -343,7 +344,7 @@ public void testTimestamps() { // also check other time units DateMathParser parser = DateFormatter.forPattern("epoch_second||date_optional_time").toDateMathParser(); - long datetime = parser.parse("1418248078", () -> 0).toEpochMilli(); + Instant datetime = parser.parse("1418248078", () -> 0).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(datetime, "1418248078", "2014-12-10T21:47:58.000"); // for date_optional_time a timestamp with more than 9digits is epoch @@ -401,14 +402,14 @@ private void assertDateMathEquals( boolean roundUp, ZoneId timeZone ) { - long gotMillis = parser.parse(toTest, () -> now, roundUp, timeZone).toEpochMilli(); + Instant gotMillis = parser.parse(toTest, () -> now, roundUp, timeZone).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(gotMillis, toTest, expected); } - private void assertDateEquals(long gotMillis, String original, String expected) { - long expectedMillis = parser.parse(expected, () -> 0).toEpochMilli(); - if (gotMillis != expectedMillis) { - ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(gotMillis), ZoneOffset.UTC); + private void assertDateEquals(Instant gotMillis, String original, String expected) { + Instant expectedMillis = parser.parse(expected, () -> 0).truncatedTo(ChronoUnit.MILLIS); + if (gotMillis.equals(expectedMillis) == false) { + ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(gotMillis, ZoneOffset.UTC); fail(Strings.format(""" Date math not equal Original : %s diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java index 86c268dd2a092..e05b67874ddbb 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.List; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.is; public class AnalysisTests extends ESTestCase { @@ -104,4 +105,92 @@ public void testParseWordList() throws IOException { List wordList = Analysis.getWordList(env, nodeSettings, "foo.bar"); assertEquals(Arrays.asList("hello", "world"), wordList); } + + public void testParseDuplicates() throws IOException { + Path tempDir = createTempDir(); + Path dict = tempDir.resolve("foo.dict"); + Settings nodeSettings = Settings.builder() + .put("foo.path", tempDir.resolve(dict)) + .put("bar.list", "") + .put("soup.lenient", "true") + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .build(); + try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) { + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞,extra stuff that gets discarded"); + writer.write('\n'); + } + Environment env = TestEnvironment.newEnvironment(nodeSettings); + List wordList = Analysis.getWordList(env, nodeSettings, "foo.path", "bar.list", "soup.lenient", true, true); + assertEquals(List.of("最終契約,最終契約,最終契約,カスタム名 詞"), wordList); + } + + public void testFailOnDuplicates() throws IOException { + Path tempDir = createTempDir(); + Path dict = tempDir.resolve("foo.dict"); + Settings nodeSettings = Settings.builder() + .put("foo.path", tempDir.resolve(dict)) + .put("bar.list", "") + .put("soup.lenient", "false") + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .build(); + try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) { + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("最終契,最終契,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞,extra"); + writer.write('\n'); + } + Environment env = TestEnvironment.newEnvironment(nodeSettings); + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> Analysis.getWordList(env, nodeSettings, "foo.path", "bar.list", "soup.lenient", false, true) + ); + assertThat(exc.getMessage(), containsString("[最終契約] in user dictionary at line [5]")); + } + + public void testParseDuplicatesWComments() throws IOException { + Path tempDir = createTempDir(); + Path dict = tempDir.resolve("foo.dict"); + Settings nodeSettings = Settings.builder() + .put("foo.path", tempDir.resolve(dict)) + .put("bar.list", "") + .put("soup.lenient", "true") + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .build(); + try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) { + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞,extra"); + writer.write('\n'); + } + Environment env = TestEnvironment.newEnvironment(nodeSettings); + List wordList = Analysis.getWordList(env, nodeSettings, "foo.path", "bar.list", "soup.lenient", false, true); + assertEquals( + List.of( + "# This is a test of the emergency broadcast system", + "最終契約,最終契約,最終契約,カスタム名 詞", + "# This is a test of the emergency broadcast system" + ), + wordList + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index b0a9a9e9fe7dc..83553503c3c5e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -268,8 +268,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", true, false)))); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", true, false); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 0e97d2b3b46ae..eaa7bf6528203 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -1518,6 +1518,37 @@ public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOEx {"path":{"at":{"foo":"A"}}}""", syntheticSource); } + public void testCopyToLogicInsideObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path"); + b.startObject("properties"); + { + b.startObject("at").field("type", "keyword").field("copy_to", "copy_top.copy").endObject(); + } + b.endObject(); + b.endObject(); + b.startObject("copy_top"); + b.startObject("properties"); + { + b.startObject("copy").field("type", "keyword").endObject(); + } + b.endObject(); + b.endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("path"); + b.field("at", "A"); + b.endObject(); + }; + + var doc = documentMapper.parse(source(document)); + assertNotNull(doc.docs().get(0).getField("copy_top.copy")); + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"path\":{\"at\":\"A\"}}", syntheticSource); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 2d190f77f60f7..86c1157259790 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -259,11 +259,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field", "192.168.1.1", "192.168.1.1"))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", "192.168.1.1", "192.168.1.1"); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 2d2fad23b3831..5b218fb077d32 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -384,8 +384,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", "1234", "45678")))); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", "1234", "45678"); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 8ba57824cf434..3312c94e8a0e1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -811,4 +811,40 @@ public void testFlattenExplicitSubobjectsTrue() { exception.getMessage() ); } + + public void testFindParentMapper() { + MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); + + var rootBuilder = new RootObjectMapper.Builder("_doc", Optional.empty()); + rootBuilder.add(new KeywordFieldMapper.Builder("keyword", IndexVersion.current())); + + var child = new ObjectMapper.Builder("child", Optional.empty()); + child.add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())); + child.add(new KeywordFieldMapper.Builder("keyword.with.dot", IndexVersion.current())); + var secondLevelChild = new ObjectMapper.Builder("child2", Optional.empty()); + secondLevelChild.add(new KeywordFieldMapper.Builder("keyword22", IndexVersion.current())); + child.add(secondLevelChild); + rootBuilder.add(child); + + var childWithDot = new ObjectMapper.Builder("childwith.dot", Optional.empty()); + childWithDot.add(new KeywordFieldMapper.Builder("keyword3", IndexVersion.current())); + childWithDot.add(new KeywordFieldMapper.Builder("keyword4.with.dot", IndexVersion.current())); + rootBuilder.add(childWithDot); + + RootObjectMapper root = rootBuilder.build(rootContext); + + assertEquals("_doc", root.findParentMapper("keyword").fullPath()); + assertNull(root.findParentMapper("aa")); + + assertEquals("child", root.findParentMapper("child.keyword2").fullPath()); + assertEquals("child", root.findParentMapper("child.keyword.with.dot").fullPath()); + assertNull(root.findParentMapper("child.long")); + assertNull(root.findParentMapper("child.long.hello")); + assertEquals("child.child2", root.findParentMapper("child.child2.keyword22").fullPath()); + + assertEquals("childwith.dot", root.findParentMapper("childwith.dot.keyword3").fullPath()); + assertEquals("childwith.dot", root.findParentMapper("childwith.dot.keyword4.with.dot").fullPath()); + assertNull(root.findParentMapper("childwith.dot.long")); + assertNull(root.findParentMapper("childwith.dot.long.hello")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index b27b8bd9766f3..9d56938f185de 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -28,10 +28,13 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -63,13 +66,19 @@ protected IndexVersion getVersion() { } private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { + return createDocumentMapper(getVersion(), routingPath, mappings); + } + + private DocumentMapper createDocumentMapper(IndexVersion version, String routingPath, XContentBuilder mappings) throws IOException { return createMapperService( + version, getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Allow tests that use many dimensions .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z") .build(), + () -> true, mappings ).documentMapper(); } @@ -644,6 +653,44 @@ public void testDifferentDimensions() throws IOException { assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } + public void testMultiValueDimensionsNotSupportedBeforeTsidHashing() throws IOException { + IndexVersion priorToTsidHashing = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_ID_HASHING); + DocumentMapper docMapper = createDocumentMapper( + priorToTsidHashing, + "a", + mapping(b -> b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject()) + ); + + String a1 = randomAlphaOfLength(10); + String a2 = randomAlphaOfLength(10); + CheckedConsumer fields = d -> d.field("a", new String[] { a1, a2 }); + DocumentParsingException exception = assertThrows(DocumentParsingException.class, () -> parseDocument(docMapper, fields)); + assertThat(exception.getMessage(), containsString("Dimension field [a] cannot be a multi-valued field")); + } + + public void testMultiValueDimensions() throws IOException { + DocumentMapper docMapper = createDocumentMapper( + IndexVersions.TIME_SERIES_ID_HASHING, + "a", + mapping(b -> b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject()) + ); + + String a1 = randomAlphaOfLength(10); + String a2 = randomAlphaOfLength(10); + List docs = List.of( + parseDocument(docMapper, d -> d.field("a", new String[] { a1 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a1, a2 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a2, a1 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a1, a2, a1 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a2, a1, a2 })) + ); + List tsids = docs.stream() + .map(doc -> doc.rootDoc().getBinaryValue("_tsid").toString()) + .distinct() + .collect(Collectors.toList()); + assertThat(tsids, hasSize(docs.size())); + } + /** * Documents with fewer dimensions have a different value. */ diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java index 4de02178beec1..f4e114da1fa51 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java @@ -240,64 +240,6 @@ private void testWriteMergedWithMixedValues(Object value, List multipleV assertEquals(expected, map.get("foo")); } - public void testWriteMergedWithVoidValue() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - XContentDataHelper.writeMerged(destination, "field", List.of(XContentDataHelper.nothing())); - - destination.endObject(); - - assertEquals("{}", Strings.toString(destination)); - } - - public void testWriteMergedWithMultipleVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.nothing(), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{}", Strings.toString(destination)); - } - - public void testWriteMergedWithMixedVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - var value = XContentFactory.contentBuilder(XContentType.JSON).value(34); - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{\"field\":34}", Strings.toString(destination)); - } - - public void testWriteMergedWithArraysAndVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - var value = XContentFactory.contentBuilder(XContentType.JSON).value(List.of(3, 4)); - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{\"field\":[3,4]}", Strings.toString(destination)); - } - private Map executeWriteMergedOnRepeated(Object value) throws IOException { return executeWriteMergedOnTwoEncodedValues(value, value); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 2f245a319f8cc..285431b881add 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import org.elasticsearch.xcontent.XContentBuilder; @@ -200,11 +201,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimensions", List.of("key1", "key2", "field3.key3")); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field.key1", "value1", "value2"))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field.key1] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field.key1", "value1", "value2"); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java index cadc836f7c4ea..f494af259c504 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java @@ -165,10 +165,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - // Nothing to do. - } } private static final ToScriptFieldFactory MOCK_TO_SCRIPT_FIELD = (dv, n) -> new DelegateDocValuesField( diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java index e9a47668faca5..946dbea45b4ba 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java @@ -126,8 +126,6 @@ public Collection getChildResources() { throw new UnsupportedOperationException(UNSUPPORTED); } - @Override - public void close() {} }; } @@ -229,8 +227,6 @@ public Collection getChildResources() { throw new UnsupportedOperationException(UNSUPPORTED); } - @Override - public void close() {} }; } diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java index b3efff4323c20..5862e1bd1329f 100644 --- a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamOutput; @@ -170,7 +171,12 @@ public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) // no work to be done on the coordinator node for the rank feature phase @Override public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from, Client client) { - return null; + return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) { + @Override + protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener scoreListener) { + throw new AssertionError("not expected"); + } + }; } @Override diff --git a/settings.gradle b/settings.gradle index ab87861105156..2926a9a303375 100644 --- a/settings.gradle +++ b/settings.gradle @@ -91,6 +91,7 @@ List projects = [ 'distribution:tools:keystore-cli', 'distribution:tools:geoip-cli', 'distribution:tools:ansi-console', + 'distribution:tools:entitlement-agent', 'server', 'test:framework', 'test:fixtures:azure-fixture', diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index bb17983264072..a297f5d13254b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -80,11 +80,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field", randomNumber(), randomNumber(), randomNumber()))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", randomNumber(), randomNumber(), randomNumber()); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index ce843fc3e15ee..81e120511a40f 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -104,9 +104,12 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) } } - record LeafMappingParametersGenerator(String fieldName, FieldType fieldType, Set eligibleCopyToFields) - implements - DataSourceRequest { + record LeafMappingParametersGenerator( + String fieldName, + FieldType fieldType, + Set eligibleCopyToFields, + DynamicMapping dynamicMapping + ) implements DataSourceRequest { public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) { return handler.handle(this); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 86cf071ce8696..89850cd56bbd0 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -48,7 +49,11 @@ private Supplier> keywordMapping( // We only add copy_to to keywords because we get into trouble with numeric fields that are copied to dynamic fields. // If first copied value is numeric, dynamic field is created with numeric field type and then copy of text values fail. // Actual value being copied does not influence the core logic of copy_to anyway. - if (ESTestCase.randomDouble() <= 0.05) { + // + // TODO + // We don't use copy_to on fields that are inside an object with dynamic: strict + // because we'll hit https://github.com/elastic/elasticsearch/issues/113049. + if (request.dynamicMapping() != DynamicMapping.FORBIDDEN && ESTestCase.randomDouble() <= 0.05) { var options = request.eligibleCopyToFields() .stream() .filter(f -> f.equals(request.fieldName()) == false) diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java index c6d9d94d61892..ba03b2f91c53c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -62,7 +62,8 @@ List generateChildFields(DynamicMapping dynamicMapping) { new DataSourceRequest.LeafMappingParametersGenerator( fieldName, fieldTypeInfo.fieldType(), - context.getEligibleCopyToDestinations() + context.getEligibleCopyToDestinations(), + dynamicMapping ) ); var generator = fieldTypeInfo.fieldType() diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java index 7b1bf4a77dc72..57e3ce3ce2a86 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java @@ -21,7 +21,7 @@ public interface PredefinedField { FieldDataGenerator generator(DataSource dataSource); - record WithType(String fieldName, FieldType fieldType) implements PredefinedField { + record WithType(String fieldName, FieldType fieldType, DynamicMapping dynamicMapping) implements PredefinedField { @Override public String name() { return fieldName; @@ -31,7 +31,7 @@ public String name() { public FieldDataGenerator generator(DataSource dataSource) { // copy_to currently not supported for predefined fields, use WithGenerator if needed var mappingParametersGenerator = dataSource.get( - new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of()) + new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of(), dynamicMapping) ); return fieldType().generator(fieldName, dataSource, mappingParametersGenerator); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 0940b3ef96ddd..d597d7b59f240 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -233,10 +233,6 @@ public long ramBytesUsed() { return 0; // Unknown } - @Override - public void close() { - - } }; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java index 1072e6ee4c899..53247d6428bfb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java @@ -9,11 +9,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Priority; import org.elasticsearch.core.Nullable; @@ -36,7 +36,7 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask { public static AckedClusterStateUpdateTask wrap( OperationModeUpdateTask task, - AckedRequest request, + AcknowledgedRequest request, ActionListener listener ) { return new AckedClusterStateUpdateTask(task.priority(), request, listener) { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json index 27262507518d2..2bf7607e86d32 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json @@ -2466,6 +2466,104 @@ } } }, + "indexing_pressure": { + "properties": { + "memory": { + "properties": { + "current": { + "properties": { + "all": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "combined_coordinating_and_primary": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "coordinating": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "primary": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "replica": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "limit_in_bytes": { + "type": "long" + }, + "total": { + "properties": { + "all": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "combined_coordinating_and_primary": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "coordinating": { + "properties": { + "bytes": { + "type": "long" + }, + "rejections": { + "type": "long" + } + } + }, + "primary": { + "properties": { + "bytes": { + "type": "long" + }, + "rejections": { + "type": "long" + } + } + }, + "replica": { + "properties": { + "bytes": { + "type": "long" + }, + "rejections": { + "type": "long" + } + } + } + } + } + } + } + } + }, "indices_stats": { "properties": { "_all": { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index b5ae35bfc8d7f..5e0e625abb914 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -46,6 +46,7 @@ import java.util.Objects; import java.util.function.IntFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -325,7 +326,11 @@ private static void checkState(boolean condition, String msg) { @Override public String toString() { - return this.getClass().getSimpleName() + "[" + "aggregators=" + aggregatorFactories + "]"; + String aggregatorDescriptions = aggregatorFactories.stream() + .map(factory -> "\"" + factory.describe() + "\"") + .collect(Collectors.joining(", ")); + + return this.getClass().getSimpleName() + "[" + "aggregators=[" + aggregatorDescriptions + "]]"; } record SegmentID(int shardIndex, int segmentIndex) { diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 44550c62bd7c5..2a882550b67b1 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -323,6 +323,35 @@ public void testProfile() throws IOException { ); } + public void testProfileOrdinalsGroupingOperator() throws IOException { + indexTimestampData(1); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | STATS AVG(value) BY test.keyword"); + builder.profile(true); + if (Build.current().isSnapshot()) { + // Lock to shard level partitioning, so we get consistent profile output + builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); + } + Map result = runEsql(builder); + + List> signatures = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + sig.add((String) o.get("operator")); + } + signatures.add(sig); + } + + assertThat(signatures.get(0).get(2), equalTo("OrdinalsGroupingOperator[aggregators=[\"sum of longs\", \"count\"]]")); + } + public void testInlineStatsProfile() throws IOException { assumeTrue("INLINESTATS only available on snapshots", Build.current().isSnapshot()); indexTimestampData(1); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index f9d83641ab4bd..a2686dfbe19a7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -367,6 +367,80 @@ date1:date | dd_ms:integer 2023-12-02T11:00:00.000Z | 1000 ; +evalDateDiffMonthAsWhole0Months + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +| EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to) +; + + from:date | to:date | msecs:integer| months:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1 |0 + +; + +evalDateDiffMonthAsWhole1Month + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to) +; + + from:date | to:date | secs:integer| months:integer +2023-12-31T23:59:59.999Z|2024-02-01T00:00:00.000Z|2678400 |1 + +; + +evalDateDiffYearAsWhole0Years +required_capability: date_diff_year_calendarial + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +| EVAL msecs=DATE_DIFF("milliseconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | msecs:integer | years:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1 |0 +; + +evalDateDiffYearAsWhole1Year +required_capability: date_diff_year_calendarial + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | secs:integer| years:integer +2023-12-31T23:59:59.999Z|2025-01-01T00:00:00.000Z|31622400 |1 +; + +evalDateDiffYearAsWhole1Year +required_capability: date_diff_year_calendarial + +ROW from="2024-01-01T00:00:00Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | secs:integer| years:integer +2024-01-01T00:00:00.000Z|2025-01-01T00:00:00.000Z|31622400 |1 +; + +evalDateDiffYearForDocs +required_capability: date_diff_year_calendarial + +// tag::evalDateDiffYearForDocs[] +ROW end_23="2023-12-31T23:59:59.999Z"::DATETIME, + start_24="2024-01-01T00:00:00.000Z"::DATETIME, + end_24="2024-12-31T23:59:59.999"::DATETIME +| EVAL end23_to_start24=DATE_DIFF("year", end_23, start_24) +| EVAL end23_to_end24=DATE_DIFF("year", end_23, end_24) +| EVAL start_to_end_24=DATE_DIFF("year", start_24, end_24) +// end::evalDateDiffYearForDocs[] +; + +// tag::evalDateDiffYearForDocs-result[] + end_23:date | start_24:date | end_24:date |end23_to_start24:integer|end23_to_end24:integer|start_to_end_24:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|2024-12-31T23:59:59.999Z|0 |1 |0 +// end::evalDateDiffYearForDocs-result[] +; + evalDateParseWithSimpleDate row a = "2023-02-01" | eval b = date_parse("yyyy-MM-dd", a) | keep b; @@ -480,6 +554,7 @@ emp_no:integer | year:long | month:long | day:long dateFormatLocale from employees | where emp_no == 10049 or emp_no == 10050 | sort emp_no | eval birth_month = date_format("MMMM", birth_date) | keep emp_no, birth_date, birth_month; +warningRegex:Date format \[MMMM\] contains textual field specifiers that could change in JDK 23 ignoreOrder:true emp_no:integer | birth_date:datetime | birth_month:keyword diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 88cbcbd608926..597c349273eb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -320,7 +320,12 @@ public enum Cap { * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. * https://github.com/elastic/elasticsearch/issues/112704 */ - FIXED_WRONG_IS_NOT_NULL_CHECK_ON_CASE; + FIXED_WRONG_IS_NOT_NULL_CHECK_ON_CASE, + + /** + * Compute year differences in full calendar years. + */ + DATE_DIFF_YEAR_CALENDARIAL; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index 582785d023945..f9039417e48a6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -66,7 +66,7 @@ public class DateDiff extends EsqlScalarFunction { */ public enum Part implements DateTimeField { - YEAR((start, end) -> end.getYear() - start.getYear(), "years", "yyyy", "yy"), + YEAR((start, end) -> safeToInt(ChronoUnit.YEARS.between(start, end)), "years", "yyyy", "yy"), QUARTER((start, end) -> safeToInt(IsoFields.QUARTER_YEARS.between(start, end)), "quarters", "qq", "q"), MONTH((start, end) -> safeToInt(ChronoUnit.MONTHS.between(start, end)), "months", "mm", "m"), DAYOFYEAR((start, end) -> safeToInt(ChronoUnit.DAYS.between(start, end)), "dy", "y"), @@ -126,36 +126,44 @@ public static Part resolve(String dateTimeUnit) { } } - @FunctionInfo(returnType = "integer", description = """ - Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`. - If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""", detailedDescription = """ - [cols=\"^,^\",role=\"styled\"] - |=== - 2+h|Datetime difference units - - s|unit - s|abbreviations - - | year | years, yy, yyyy - | quarter | quarters, qq, q - | month | months, mm, m - | dayofyear | dy, y - | day | days, dd, d - | week | weeks, wk, ww - | weekday | weekdays, dw - | hour | hours, hh - | minute | minutes, mi, n - | second | seconds, ss, s - | millisecond | milliseconds, ms - | microsecond | microseconds, mcs - | nanosecond | nanoseconds, ns - |=== - - Note that while there is an overlap between the function's supported units and - {esql}'s supported time span literals, these sets are distinct and not - interchangeable. Similarly, the supported abbreviations are conveniently shared - with implementations of this function in other established products and not - necessarily common with the date-time nomenclature used by {es}.""", examples = @Example(file = "date", tag = "docsDateDiff")) + @FunctionInfo( + returnType = "integer", + description = """ + Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`. + If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""", + detailedDescription = """ + [cols=\"^,^\",role=\"styled\"] + |=== + 2+h|Datetime difference units + + s|unit + s|abbreviations + + | year | years, yy, yyyy + | quarter | quarters, qq, q + | month | months, mm, m + | dayofyear | dy, y + | day | days, dd, d + | week | weeks, wk, ww + | weekday | weekdays, dw + | hour | hours, hh + | minute | minutes, mi, n + | second | seconds, ss, s + | millisecond | milliseconds, ms + | microsecond | microseconds, mcs + | nanosecond | nanoseconds, ns + |=== + + Note that while there is an overlap between the function's supported units and + {esql}'s supported time span literals, these sets are distinct and not + interchangeable. Similarly, the supported abbreviations are conveniently shared + with implementations of this function in other established products and not + necessarily common with the date-time nomenclature used by {es}.""", + examples = { @Example(file = "date", tag = "docsDateDiff"), @Example(description = """ + When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted. + To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly. + """, file = "date", tag = "evalDateDiffYearForDocs") } + ) public DateDiff( Source source, @Param(name = "unit", type = { "keyword", "text" }, description = "Time difference unit") Expression unit, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java index 81f391d637317..4dbdfd1e56854 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java @@ -113,6 +113,34 @@ public static Iterable parameters() { ) ) ); + suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { + ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z"); + ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"), + new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), + new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") + ), + "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]", + DataType.INTEGER, + equalTo(1) + ); + })); + suppliers.add(new TestCaseSupplier("Date Diff In Year - 0", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { + ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01.001Z"); + ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"), + new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), + new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") + ), + "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]", + DataType.INTEGER, + equalTo(0) + ); + })); return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index b1f38191ba7b3..6944f91042311 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -469,8 +469,6 @@ public long ramBytesUsed() { return 0; // Unknown } - @Override - public void close() {} }; } diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java index e728a74955f30..e52237f4d507e 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java @@ -155,10 +155,6 @@ public long ramBytesUsed() { return 0; // Unknown } - @Override - public void close() { - // nothing to close - } }; } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index 0c98c5b8bd5a7..00c60987f439d 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.fielddata.plain.FormattedSortedNumericDocValues; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.script.field.ToScriptFieldFactory; import org.elasticsearch.search.DocValueFormat; @@ -97,23 +98,7 @@ public void close() { @Override public FormattedDocValues getFormattedValues(DocValueFormat format) { - SortedNumericDocValues values = getLongValues(); - return new FormattedDocValues() { - @Override - public boolean advanceExact(int docId) throws IOException { - return values.advanceExact(docId); - } - - @Override - public int docValueCount() { - return values.docValueCount(); - } - - @Override - public Object nextValue() throws IOException { - return format.format(values.nextValue()); - } - }; + return new FormattedSortedNumericDocValues(getLongValues(), format); } static double convertUnsignedLongToDouble(long value) { diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index cdb25fbe995b2..f554a84048fde 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.NumberTypeOutOfRangeSpec; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.TimeSeriesParams; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.WholeNumberFieldMapperTests; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xcontent.XContentBuilder; @@ -268,11 +269,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field", randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index 89ff28249aabb..a4413d266181d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -28,6 +28,11 @@ template: type: constant_keyword value: metrics dynamic_templates: + - ecs_ip: + mapping: + type: ip + path_match: [ "ip", "*.ip", "*_ip" ] + match_mapping_type: string - all_strings_to_keywords: mapping: ignore_above: 1024 diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml index a6591d6c32210..1823dfab7e716 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml @@ -123,11 +123,10 @@ setup: start_time: 2024-07-01T13:03:08.138Z mappings: dynamic_templates: - - ip_fields: + - no_ip_fields: mapping: - type: ip + type: keyword match_mapping_type: string - path_match: "*.ip" - do: bulk: index: metrics-generic.otel-default @@ -145,5 +144,37 @@ setup: indices.get_mapping: index: $idx0name expand_wildcards: hidden - - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' } + - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'keyword' } - match: { .$idx0name.mappings.properties.attributes.properties.foo.type: "keyword" } +--- +IP dimensions: + - requires: + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {"dynamic_templates":{"metrics.foo.bar":"counter_long"}} + - "@timestamp": 2024-07-18T14:48:33.467654000Z + resource: + attributes: + host.ip: [ "127.0.0.1", "0.0.0.0" ] + attributes: + philip: [ a, b, c ] + metrics: + foo.bar: 42 + - is_false: errors + + - do: + indices.get_data_stream: + name: metrics-generic.otel-default + - set: { data_streams.0.indices.0.index_name: idx0name } + + - do: + indices.get_mapping: + index: $idx0name + expand_wildcards: hidden + - match: { .$idx0name.mappings.properties.resource.properties.attributes.properties.host\.ip.type: 'ip' } + - match: { .$idx0name.mappings.properties.attributes.properties.philip.type: "keyword" } diff --git a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java index 595d48ea92a44..b7fb8c37f4c45 100644 --- a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java +++ b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java @@ -296,7 +296,8 @@ public class ServiceAccountIT extends ESRestTestCase { "monitor", "create_index", "auto_configure", - "maintenance" + "maintenance", + "view_index_metadata" ], "allow_restricted_indices": false }, @@ -311,7 +312,8 @@ public class ServiceAccountIT extends ESRestTestCase { "monitor", "create_index", "auto_configure", - "maintenance" + "maintenance", + "view_index_metadata" ], "allow_restricted_indices": false } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index f4c3b77af3abe..28c75987248ee 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -36,7 +36,7 @@ public abstract class NativeRealmIntegTestCase extends SecurityIntegTestCase { @Before public void ensureNativeStoresStarted() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); if (shouldSetReservedUserPasswords()) { setupReservedPasswords(); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index f164189c6f047..70dcfbaa315cf 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -191,7 +191,7 @@ protected boolean addMockHttpTransport() { @Before public void waitForSecurityIndexWritable() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); } @After diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index 66ea1235800ed..fef1a98ca67e9 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -783,7 +783,7 @@ public void testAuthenticateWithWrongToken() throws Exception { @Before public void waitForSecurityIndexWritable() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); } @After diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java index 1b62c79236a9c..eb7c5e5276c15 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java @@ -31,7 +31,7 @@ public class SecurityScrollTests extends SecurityIntegTestCase { public void testScrollIsPerUser() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); new PutRoleRequestBuilder(client()).name("scrollable") .addIndices(new String[] { randomAlphaOfLengthBetween(4, 12) }, new String[] { "read" }, null, null, null, randomBoolean()) .get(); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java index 4bbffc139cbfd..32337f0d66896 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java @@ -46,7 +46,6 @@ public class SecurityIndexManagerIntegTests extends SecurityIntegTestCase { public void testConcurrentOperationsTryingToCreateSecurityIndexAndAlias() throws Exception { - assertSecurityIndexActive(); final int processors = Runtime.getRuntime().availableProcessors(); final int numThreads = Math.min(50, scaledRandomIntBetween((processors + 1) / 2, 4 * processors)); // up to 50 threads final int maxNumRequests = 50 / numThreads; // bound to a maximum of 50 requests @@ -111,7 +110,7 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex // pick longer wait than in the assertBusy that waits for below to ensure index has had enough time to initialize securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueSeconds(40)); - createSecurityIndex(); + createSecurityIndexWithWaitForActiveShards(); assertBusy( () -> assertThat(securityIndexManager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), is(true)), @@ -126,7 +125,7 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex @SuppressWarnings("unchecked") public void testOnIndexAvailableForSearchIndexAlreadyAvailable() throws Exception { - createSecurityIndex(); + createSecurityIndexWithWaitForActiveShards(); final SecurityIndexManager securityIndexManager = internalCluster().getInstances(NativePrivilegeStore.class) .iterator() diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java index baa920eee275b..67211ec6135be 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java @@ -160,12 +160,12 @@ final class ElasticServiceAccounts { // Custom permissions required for running Elastic connectors integration RoleDescriptor.IndicesPrivileges.builder() .indices(".elastic-connectors*") - .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance") + .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance", "view_index_metadata") .build(), // Permissions for data indices and access control filters used by Elastic connectors integration RoleDescriptor.IndicesPrivileges.builder() .indices("content-*", ".search-acl-filter-*") - .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance") + .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance", "view_index_metadata") .build(), }, new RoleDescriptor.ApplicationResourcePrivileges[] { RoleDescriptor.ApplicationResourcePrivileges.builder() diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index ddcef0e24727d..3a39d54567726 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.internal.Client; @@ -381,6 +382,12 @@ protected Function getClientWrapper() { return client -> (client instanceof NodeClient) ? client.filterWithHeader(headers) : client; } + /** + * Waits for security index to become available. Note that you must ensure index creation was triggered before calling this method, + * by calling one of the resource creation APIs (e.g., creating a user). + * If you use {@link #createSecurityIndexWithWaitForActiveShards()} to create the index it's not necessary to call + * {@link #assertSecurityIndexActive} since the create method ensures the index is active. + */ public void assertSecurityIndexActive() throws Exception { assertSecurityIndexActive(cluster()); } @@ -391,14 +398,10 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception ClusterState clusterState = client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState(); assertFalse(clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)); Index securityIndex = resolveSecurityIndex(clusterState.metadata()); - // TODO this is a bug -- since we are not tripping assertions here, this will complete successfully even if the security - // index does not exist - if (securityIndex != null) { - IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); - if (indexRoutingTable != null) { - assertTrue(indexRoutingTable.allPrimaryShardsActive()); - } - } + assertNotNull(securityIndex); + IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); + assertNotNull(indexRoutingTable); + assertTrue(indexRoutingTable.allPrimaryShardsActive()); }, 30L, TimeUnit.SECONDS); } } @@ -424,7 +427,7 @@ protected void deleteSecurityIndex() { } } - protected void createSecurityIndex() { + protected void createSecurityIndexWithWaitForActiveShards() { final Client client = client().filterWithHeader( Collections.singletonMap( "Authorization", @@ -434,7 +437,8 @@ protected void createSecurityIndex() { ) ) ); - CreateIndexRequest createIndexRequest = new CreateIndexRequest(SECURITY_MAIN_ALIAS); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(SECURITY_MAIN_ALIAS).waitForActiveShards(ActiveShardCount.ALL) + .masterNodeTimeout(TEST_REQUEST_TIMEOUT); client.admin().indices().create(createIndexRequest).actionGet(); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java index f3a3b4d2f3f27..8be44242886c9 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java @@ -38,9 +38,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} - @Override public T getShapeValues() { return emptyValues; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java index 9cbbe04f56cd5..b8bd5e82fef3b 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java @@ -34,11 +34,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public SortedNumericDocValues getSortedNumericDocValues() { try { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java index 34e6c569a4e68..42eb1925601cd 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java @@ -37,11 +37,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public CartesianShapeValues getShapeValues() { try { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java index 4790924412cd7..bf4202466eb41 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java @@ -70,10 +70,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } }; } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java index 8319da1306be6..a1aedb07825f3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java @@ -33,11 +33,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public GeoShapeValues getShapeValues() { try { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java index 5d916b6c5cead..68d5e3bd04421 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java @@ -9,18 +9,14 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.common.Rounding; import org.elasticsearch.index.fielddata.DocValueBits; import org.elasticsearch.index.fielddata.MultiPointValues; -import org.elasticsearch.index.fielddata.PointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.AggregationErrors; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.xcontent.ToXContentFragment; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.spatial.common.CartesianPoint; import org.elasticsearch.xpack.spatial.index.fielddata.IndexCartesianPointFieldData; @@ -73,33 +69,6 @@ public CartesianPoint nextValue() throws IOException { return point.resetFromEncoded(numericValues.nextValue()); } - @Override - protected PointValues getPointValues() { - final NumericDocValues singleton = DocValues.unwrapSingleton(numericValues); - return singleton != null ? new CartesianPointValues(singleton) : null; - } - } - - public static final class CartesianPointValues extends PointValues { - - private final CartesianPoint point = new CartesianPoint(); - - CartesianPointValues(NumericDocValues values) { - super(values); - } - - @Override - public CartesianPoint pointValue() throws IOException { - return point.resetFromEncoded(values.longValue()); - } - } - - public static final class CartesianPointValue implements ToXContentFragment { - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return null; - } } /** diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java index c03f853e72f88..9a9de86358551 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java @@ -6,11 +6,9 @@ */ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.xpack.sql.AbstractSqlWireSerializingTestCase; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NamedDateTimeProcessor.NameExtractor; -import org.junit.Assume; import java.time.ZoneId; @@ -45,7 +43,6 @@ protected ZoneId instanceZoneId(NamedDateTimeProcessor instance) { } public void testValidDayNamesInUTC() { - assumeCompatLocaleProviderSetting(); NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.DAY_NAME, UTC); assertEquals("Thursday", proc.process(dateTime(0L))); assertEquals("Saturday", proc.process(dateTime(-64164233612338L))); @@ -58,7 +55,6 @@ public void testValidDayNamesInUTC() { } public void testValidDayNamesWithNonUTCTimeZone() { - assumeCompatLocaleProviderSetting(); NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.DAY_NAME, ZoneId.of("GMT-10:00")); assertEquals("Wednesday", proc.process(dateTime(0))); assertEquals("Friday", proc.process(dateTime(-64164233612338L))); @@ -72,7 +68,6 @@ public void testValidDayNamesWithNonUTCTimeZone() { } public void testValidMonthNamesInUTC() { - assumeCompatLocaleProviderSetting(); NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.MONTH_NAME, UTC); assertEquals("January", proc.process(dateTime(0))); assertEquals("September", proc.process(dateTime(-64165813612338L))); @@ -85,7 +80,6 @@ public void testValidMonthNamesInUTC() { } public void testValidMonthNamesWithNonUTCTimeZone() { - assumeCompatLocaleProviderSetting(); NamedDateTimeProcessor proc = new NamedDateTimeProcessor(NameExtractor.MONTH_NAME, ZoneId.of("GMT-03:00")); assertEquals("December", proc.process(dateTime(0))); assertEquals("August", proc.process(dateTime(-64165813612338L))); // GMT: Tuesday, September 1, -0064 2:53:07.662 AM @@ -97,19 +91,4 @@ public void testValidMonthNamesWithNonUTCTimeZone() { assertEquals("July", proc.process(dateTime(10902, 8, 1, 2, 59))); assertEquals("August", proc.process(dateTime(10902, 8, 1, 3, 00))); } - - /* - * This method checks the existence of a jvm parameter that should exist in ES jvm.options for Java 9+. If the parameter is - * missing, the tests will be skipped. Not doing this, the tests will fail because the day and month names will be in the narrow - * format (Mon, Tue, Jan, Feb etc) instead of full format (Monday, Tuesday, January, February etc). - * - * Related infra issue: https://github.com/elastic/elasticsearch/issues/33796 - */ - private void assumeCompatLocaleProviderSetting() { - String beforeJava9CompatibleLocale = System.getProperty("java.locale.providers"); - // and COMPAT setting needs to be first on the list - boolean isBeforeJava9Compatible = beforeJava9CompatibleLocale != null - && Strings.tokenizeToStringArray(beforeJava9CompatibleLocale, ",")[0].equals("COMPAT"); - Assume.assumeTrue(isBeforeJava9Compatible); - } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index e181f77f2bcef..05edf6cdfb5a8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -29,6 +29,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' @@ -50,6 +51,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml new file mode 100644 index 0000000000000..545b9ef1e0285 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml @@ -0,0 +1,31 @@ +--- +"Test basic response with invalid credentials": + + - skip: + features: headers + + - do: + headers: + Authorization: "Basic dGVzdF91c2VyOndyb25nLXBhc3N3b3Jk" # invalid credentials + info: {} + catch: unauthorized + + - match: + error.root_cause.0.type: security_exception + +--- +"Test bulk response with invalid credentials": + + - skip: + features: headers + + - do: + headers: + Authorization: "Basic dGVzdF91c2VyOndyb25nLXBhc3N3b3Jk" # invalid credentials + bulk: + body: | + {"index": {}} + {} + catch: unauthorized + - match: + error.root_cause.0.type: security_exception diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml index 1d370082c8e48..e1b297f1b5d78 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml @@ -90,3 +90,53 @@ setup: - match: {hits.total: 1 } - length: {hits.hits: 1 } - match: {hits.hits.0._id: "1" } + +--- +"Failed to snapshot indices with synthetic source": + - skip: + features: ["allowed_warnings"] + + - do: + indices.create: + index: test_synthetic + body: + mappings: + _source: + mode: synthetic + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + snapshot.create: + repository: test_repo_restore_1 + snapshot: test_snapshot_2 + wait_for_completion: true + body: | + { "indices": "test_synthetic" } + + - match: { snapshot.snapshot: test_snapshot_2 } + - match: { snapshot.state : PARTIAL } + - match: { snapshot.shards.successful: 0 } + - match: { snapshot.shards.failed : 1 } + - match: { snapshot.failures.0.index: "test_synthetic" } + - match: { snapshot.failures.0.reason : "IllegalStateException[Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source]" } + - is_true: snapshot.version + - gt: { snapshot.version_id: 0} + + - do: + snapshot.create: + repository: test_repo_restore_1 + snapshot: test_snapshot_3 + wait_for_completion: true + body: | + { "indices": "test_*" } + + - match: { snapshot.snapshot: test_snapshot_3 } + - match: { snapshot.state : PARTIAL } + - match: { snapshot.shards.successful: 1 } + - match: { snapshot.shards.failed : 1 } + - match: { snapshot.failures.0.index: "test_synthetic" } + - match: { snapshot.failures.0.reason: "IllegalStateException[Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source]" } + - is_true: snapshot.version + - gt: { snapshot.version_id: 0} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java index 9212780d11fd3..5e6363b993ce4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java @@ -16,14 +16,12 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -69,42 +67,35 @@ protected void masterOperation( final boolean manuallyStopped = request.getCommand() == WatcherServiceRequest.Command.STOP; final String source = manuallyStopped ? "update_watcher_manually_stopped" : "update_watcher_manually_started"; - // TODO: make WatcherServiceRequest a real AckedRequest so that we have both a configurable timeout and master node timeout like - // we do elsewhere - submitUnbatchedTask(source, new AckedClusterStateUpdateTask(new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; - } - - @Override - public TimeValue masterNodeTimeout() { - return request.masterNodeTimeout(); - } - }, listener) { - @Override - public ClusterState execute(ClusterState clusterState) { - XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); + // TODO: make WatcherServiceRequest a real AcknowledgedRequest so that we have both a configurable timeout and master node timeout + // like we do elsewhere + submitUnbatchedTask( + source, + new AckedClusterStateUpdateTask(request.masterNodeTimeout(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, listener) { + @Override + public ClusterState execute(ClusterState clusterState) { + XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); - WatcherMetadata newWatcherMetadata = new WatcherMetadata(manuallyStopped); - WatcherMetadata currentMetadata = clusterState.metadata().custom(WatcherMetadata.TYPE); + WatcherMetadata newWatcherMetadata = new WatcherMetadata(manuallyStopped); + WatcherMetadata currentMetadata = clusterState.metadata().custom(WatcherMetadata.TYPE); - // adhere to the contract of returning the original state if nothing has changed - if (newWatcherMetadata.equals(currentMetadata)) { - return clusterState; - } else { - ClusterState.Builder builder = new ClusterState.Builder(clusterState); - builder.metadata(Metadata.builder(clusterState.getMetadata()).putCustom(WatcherMetadata.TYPE, newWatcherMetadata)); - return builder.build(); + // adhere to the contract of returning the original state if nothing has changed + if (newWatcherMetadata.equals(currentMetadata)) { + return clusterState; + } else { + ClusterState.Builder builder = new ClusterState.Builder(clusterState); + builder.metadata(Metadata.builder(clusterState.getMetadata()).putCustom(WatcherMetadata.TYPE, newWatcherMetadata)); + return builder.build(); + } } - } - @Override - public void onFailure(Exception e) { - logger.error(() -> format("could not update watcher stopped status to [%s], source [%s]", manuallyStopped, source), e); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.error(() -> format("could not update watcher stopped status to [%s], source [%s]", manuallyStopped, source), e); + listener.onFailure(e); + } } - }); + ); } @Override diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java index 17363d58545c2..87c7e344336ad 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java @@ -181,7 +181,7 @@ protected Settings buildRealmSettings( @Before public void setupRoleMappings() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); List content = getRoleMappingContent(RoleMappingEntry::nativeContent); if (content.isEmpty()) {