Merge branch 'main' into enhancement/match_no_docs_span_unmapped

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -168,7 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,CLDR',
+              '-Djava.locale.providers=CLDR',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -93,7 +93,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,CLDR");
+                    test.systemProperty("java.locale.providers", "CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.nativeaccess",
         "org.elasticsearch.plugin",
         "org.elasticsearch.plugin.analysis",
-        "org.elasticsearch.pluginclassloader",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
         "org.elasticsearch.simdvec",

distribution/tools/entitlement-agent/README.md

@@ -0,0 +1,10 @@
+### Entitlement Agent
+
+This is a java agent that instruments sensitive class library methods with calls into the `entitlement-runtime` module to check for permissions granted under the _entitlements_ system.
+
+The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
+With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers.
+
+This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the `entitlement-runtime` module.
+
+It is not responsible for permission granting or checking logic. That responsibility lies with `entitlement-runtime`.

distribution/tools/entitlement-agent/build.gradle

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+apply plugin: 'elasticsearch.build'
+
+configurations {
+  entitlementRuntime
+}
+
+dependencies {
+  entitlementRuntime project(":libs:elasticsearch-entitlement-runtime")
+  implementation project(":libs:elasticsearch-entitlement-runtime")
+  testImplementation project(":test:framework")
+}
+
+tasks.named('test').configure {
+  dependsOn('jar')
+  jvmArgs "-javaagent:${ tasks.named('jar').flatMap{ it.archiveFile }.get()}"
+}
+
+tasks.named('jar').configure {
+  manifest {
+    attributes(
+      'Premain-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent'
+      , 'Can-Retransform-Classes': 'true'
+    )
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+

libs/plugin-classloader/src/main/java/module-info.java → distribution/tools/entitlement-agent/src/main/java/module-info.java

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-module org.elasticsearch.pluginclassloader {
-    exports org.elasticsearch.plugins.loader;
+module org.elasticsearch.entitlement.agent {
+    requires java.instrument;
+    requires org.elasticsearch.entitlement.runtime;
 }

server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java → distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java

@@ -7,22 +7,15 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.cluster.ack;
+package org.elasticsearch.entitlement.agent;
 
-import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
 
-/**
- * Identifies a cluster state update request with acknowledgement support
- */
-public interface AckedRequest {
+import java.lang.instrument.Instrumentation;
 
-    /**
-     * Returns the acknowledgement timeout
-     */
-    TimeValue ackTimeout();
+public class EntitlementAgent {
 
-    /**
-     * Returns the timeout for the request to be completed on the master node
-     */
-    TimeValue masterNodeTimeout();
+    public static void premain(String agentArgs, Instrumentation inst) throws Exception {
+        EntitlementChecks.setAgentBooted();
+    }
 }

distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.agent;
+
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
+
+/**
+ * This is an end-to-end test that runs with the javaagent installed.
+ * It should exhaustively test every instrumented method to make sure it passes with the entitlement
+ * and fails without it.
+ * See {@code build.gradle} for how we set the command line arguments for this test.
+ */
+@WithoutSecurityManager
+public class EntitlementAgentTests extends ESTestCase {
+
+    public void testAgentBooted() {
+        assertTrue(EntitlementChecks.isAgentBooted());
+    }
+
+}

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.core.UpdateForV9;
 
 import java.util.List;
 import java.util.Map;
@@ -61,7 +60,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
-                "-Djava.locale.providers=" + getLocaleProviders(),
+                "-Djava.locale.providers=CLDR",
                 maybeEnableNativeAccess(),
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),
@@ -73,16 +72,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
-    @UpdateForV9    // only use CLDR in v9+
-    private static String getLocaleProviders() {
-        /*
-         * Specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday.
-         * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can.
-         * When on JDK 23+, use the default CLDR locale database, as COMPAT was removed in JDK 23.
-         */
-        return Runtime.version().feature() >= 23 ? "SPI,CLDR" : "SPI,COMPAT";
-    }
-
     /*
      * The virtual file /proc/self/cgroup should list the current cgroup
      * membership. For each hierarchy, you can follow the cgroup path from

docs/changelog/112645.yaml

@@ -0,0 +1,6 @@
+pr: 112645
+summary: Add support for multi-value dimensions
+area: Mapping
+type: enhancement
+issues:
+ - 110387

docs/changelog/112768.yaml

@@ -0,0 +1,5 @@
+pr: 112768
+summary: Deduplicate Kuromoji User Dictionary
+area: Search
+type: enhancement
+issues: []

docs/changelog/113102.yaml

@@ -0,0 +1,5 @@
+pr: 113102
+summary: Trigger merges after recovery
+area: Recovery
+type: enhancement
+issues: []

docs/changelog/113103.yaml

@@ -0,0 +1,6 @@
+pr: 113103
+summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological"
+area: ES|QL
+type: bug
+issues:
+ - 112482

docs/plugins/analysis-kuromoji.asciidoc

@@ -133,6 +133,11 @@ unknown words. It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -221,7 +226,8 @@ PUT kuromoji_sample
             "type": "kuromoji_tokenizer",
             "mode": "extended",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ja.txt"
+            "user_dictionary": "userdict_ja.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {

docs/plugins/analysis-nori.asciidoc

@@ -58,6 +58,11 @@ It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -104,7 +109,8 @@ PUT nori_sample
             "type": "nori_tokenizer",
             "decompound_mode": "mixed",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ko.txt"
+            "user_dictionary": "userdict_ko.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {
@@ -299,7 +305,6 @@ Which responds with:
 }
 --------------------------------------------------
 
-
 [[analysis-nori-speech]]
 ==== `nori_part_of_speech` token filter
 

docs/reference/inference/service-elasticsearch.asciidoc

@@ -179,6 +179,7 @@ PUT _inference/text_embedding/my-e5-model
       "min_number_of_allocations": 3,
       "max_number_of_allocations": 10
     },
+    "num_threads": 1,
     "model_id": ".multilingual-e5-small"
   }
 }

docs/reference/inference/service-elser.asciidoc

@@ -147,7 +147,8 @@ PUT _inference/sparse_embedding/my-elser-model
       "enabled": true,
       "min_number_of_allocations": 3,
       "max_number_of_allocations": 10
-    }
+    },
+    "num_threads": 1
   }
 }
 ------------------------------------------------------------

docs/reference/mapping/types/keyword.asciidoc

@@ -163,7 +163,6 @@ index setting limits the number of dimensions in an index.
 Dimension fields have the following constraints:
 
 * The `doc_values` and `index` mapping parameters must be `true`.
-* Field values cannot be an <<array,array or multi-value>>.
 // end::dimension[]
 * Dimension values are used to identify a document’s time series. If dimension values are altered in any way during indexing, the document will be stored as belonging to different from intended time series. As a result there are additional constraints:
 ** The field cannot use a <<normalizer,`normalizer`>>.

docs/reference/query-dsl/sparse-vector-query.asciidoc

@@ -104,7 +104,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/query-dsl/text-expansion-query.asciidoc

@@ -68,7 +68,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/query-dsl/weighted-tokens-query.asciidoc

@@ -58,7 +58,7 @@ This value must between 1 and 100.
 Default: `5`.
 
 `tokens_weight_threshold`::
-(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/release-notes/8.15.0.asciidoc

@@ -35,6 +35,7 @@ can be configured using the https://www.elastic.co/guide/en/elasticsearch/refere
 ** These indices have many conflicting field mappings
 ** Many of those fields are included in the request
 These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]).
+In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard.
 +
 To work around this issue, you have a number of options:
 ** Downgrade to an earlier version
@@ -43,6 +44,14 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
+* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or
+partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved.
+There is no workaround and the only option to is to upgrade to 8.15.2 when released.
++
+If you use synthetic source then you may be affected by this bug if the following is true:
+** If you have more fields then the `index.mapping.total_fields.limit` setting allows.
+** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled.
+
 [[breaking-8.15.0]]
 [float]
 === Breaking changes