Merge branch 'main' into exhaustive-kerb-locale-fix

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.nativeaccess",
         "org.elasticsearch.plugin",
         "org.elasticsearch.plugin.analysis",
-        "org.elasticsearch.pluginclassloader",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
         "org.elasticsearch.simdvec",

distribution/tools/entitlement-agent/README.md

@@ -0,0 +1,10 @@
+### Entitlement Agent
+
+This is a java agent that instruments sensitive class library methods with calls into the `entitlement-runtime` module to check for permissions granted under the _entitlements_ system.
+
+The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
+With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers.
+
+This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the `entitlement-runtime` module.
+
+It is not responsible for permission granting or checking logic. That responsibility lies with `entitlement-runtime`.

distribution/tools/entitlement-agent/build.gradle

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+apply plugin: 'elasticsearch.build'
+
+configurations {
+  entitlementRuntime
+}
+
+dependencies {
+  entitlementRuntime project(":libs:elasticsearch-entitlement-runtime")
+  implementation project(":libs:elasticsearch-entitlement-runtime")
+  testImplementation project(":test:framework")
+}
+
+tasks.named('test').configure {
+  dependsOn('jar')
+  jvmArgs "-javaagent:${ tasks.named('jar').flatMap{ it.archiveFile }.get()}"
+}
+
+tasks.named('jar').configure {
+  manifest {
+    attributes(
+      'Premain-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent'
+      , 'Can-Retransform-Classes': 'true'
+    )
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+

libs/plugin-classloader/src/main/java/module-info.java → distribution/tools/entitlement-agent/src/main/java/module-info.java

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-module org.elasticsearch.pluginclassloader {
-    exports org.elasticsearch.plugins.loader;
+module org.elasticsearch.entitlement.agent {
+    requires java.instrument;
+    requires org.elasticsearch.entitlement.runtime;
 }

distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.agent;
+
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
+
+import java.lang.instrument.Instrumentation;
+
+public class EntitlementAgent {
+
+    public static void premain(String agentArgs, Instrumentation inst) throws Exception {
+        EntitlementChecks.setAgentBooted();
+    }
+}

distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.agent;
+
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
+
+/**
+ * This is an end-to-end test that runs with the javaagent installed.
+ * It should exhaustively test every instrumented method to make sure it passes with the entitlement
+ * and fails without it.
+ * See {@code build.gradle} for how we set the command line arguments for this test.
+ */
+@WithoutSecurityManager
+public class EntitlementAgentTests extends ESTestCase {
+
+    public void testAgentBooted() {
+        assertTrue(EntitlementChecks.isAgentBooted());
+    }
+
+}

docs/changelog/112645.yaml

@@ -0,0 +1,6 @@
+pr: 112645
+summary: Add support for multi-value dimensions
+area: Mapping
+type: enhancement
+issues:
+ - 110387

docs/changelog/112768.yaml

@@ -0,0 +1,5 @@
+pr: 112768
+summary: Deduplicate Kuromoji User Dictionary
+area: Search
+type: enhancement
+issues: []

docs/changelog/113102.yaml

@@ -0,0 +1,5 @@
+pr: 113102
+summary: Trigger merges after recovery
+area: Recovery
+type: enhancement
+issues: []

docs/plugins/analysis-kuromoji.asciidoc

@@ -133,6 +133,11 @@ unknown words. It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -221,7 +226,8 @@ PUT kuromoji_sample
             "type": "kuromoji_tokenizer",
             "mode": "extended",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ja.txt"
+            "user_dictionary": "userdict_ja.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {

docs/plugins/analysis-nori.asciidoc

@@ -58,6 +58,11 @@ It can be set to:
 
     Whether punctuation should be discarded from the output. Defaults to `true`.
 
+`lenient`::
+
+    Whether the `user_dictionary` should be deduplicated on the provided `text`.
+    False by default causing duplicates to generate an error.
+
 `user_dictionary`::
 +
 --
@@ -104,7 +109,8 @@ PUT nori_sample
             "type": "nori_tokenizer",
             "decompound_mode": "mixed",
             "discard_punctuation": "false",
-            "user_dictionary": "userdict_ko.txt"
+            "user_dictionary": "userdict_ko.txt",
+            "lenient": "true"
           }
         },
         "analyzer": {
@@ -299,7 +305,6 @@ Which responds with:
 }
 --------------------------------------------------
 
-
 [[analysis-nori-speech]]
 ==== `nori_part_of_speech` token filter
 

docs/reference/mapping/types/keyword.asciidoc

@@ -163,7 +163,6 @@ index setting limits the number of dimensions in an index.
 Dimension fields have the following constraints:
 
 * The `doc_values` and `index` mapping parameters must be `true`.
-* Field values cannot be an <<array,array or multi-value>>.
 // end::dimension[]
 * Dimension values are used to identify a document’s time series. If dimension values are altered in any way during indexing, the document will be stored as belonging to different from intended time series. As a result there are additional constraints:
 ** The field cannot use a <<normalizer,`normalizer`>>.

docs/reference/query-dsl/sparse-vector-query.asciidoc

@@ -104,7 +104,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/query-dsl/text-expansion-query.asciidoc

@@ -68,7 +68,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/query-dsl/weighted-tokens-query.asciidoc

@@ -58,7 +58,7 @@ This value must between 1 and 100.
 Default: `5`.
 
 `tokens_weight_threshold`::
-(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

docs/reference/release-notes/8.15.0.asciidoc

@@ -44,6 +44,14 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
+* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or
+partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved.
+There is no workaround and the only option to is to upgrade to 8.15.2 when released.
++
+If you use synthetic source then you may be affected by this bug if the following is true:
+** If you have more fields then the `index.mapping.total_fields.limit` setting allows.
+** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled.
+
 [[breaking-8.15.0]]
 [float]
 === Breaking changes

docs/reference/release-notes/8.15.1.asciidoc

@@ -24,13 +24,20 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
-* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field 
+* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field
 but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds:
 ** If the affected index already contains documents with a dense vector field, force merge the index to a single segment.
 ** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field
 and then force merge to a single segment.
 ** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields.
 
+* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or
+partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved.
+There is no workaround and the only option to is to upgrade to 8.15.2 when released.
++
+If you use synthetic source then you may be affected by this bug if the following is true:
+** If you have more fields then the `index.mapping.total_fields.limit` setting allows.
+** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled.
 
 [[bug-8.15.1]]
 [float]

docs/reference/rest-api/security/get-service-accounts.asciidoc

@@ -250,7 +250,8 @@ GET /_security/service/elastic/fleet-server
             "monitor",
             "create_index",
             "auto_configure",
-            "maintenance"
+            "maintenance",
+            "view_index_metadata"
           ],
           "allow_restricted_indices": false
         },
@@ -265,7 +266,8 @@ GET /_security/service/elastic/fleet-server
             "monitor",
             "create_index",
             "auto_configure",
-            "maintenance"
+            "maintenance",
+            "view_index_metadata"
           ],
           "allow_restricted_indices": false
         }

libs/entitlement-runtime/README.md

@@ -0,0 +1,14 @@
+### Entitlement runtime
+
+This module implements mechanisms to grant and check permissions under the _entitlements_ system.
+
+The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
+The `entitlement-agent` tool instruments sensitive class library methods with calls to this module, in order to enforce the controls.
+
+This module is responsible for:
+- Defining which class library methods are sensitive
+- Defining what permissions should be checked for each sensitive method
+- Implementing the permission checks
+- Offering a "grant" API to grant permissions
+
+It is not responsible for anything to do with bytecode instrumentation; that responsibility lies with `entitlement-agent`.

libs/plugin-classloader/build.gradle → libs/entitlement-runtime/build.gradle

@@ -6,13 +6,19 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
-// This is only required because :server needs this at runtime.
-// We'll be removing this in 8.0 so for now just publish the JAR to make dependency resolution work.
+apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 
-tasks.named("test").configure { enabled = false }
+dependencies {
+  compileOnly project(':libs:elasticsearch-core')
+
+  testImplementation project(":test:framework")
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
 
-// test depend on ES core...
-tasks.named('forbiddenApisMain').configure { enabled = false}
-tasks.named("jarHell").configure { enabled = false }
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}

libs/entitlement-runtime/src/main/java/module-info.java

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module org.elasticsearch.entitlement.runtime {
+    requires org.elasticsearch.base;
+
+    exports org.elasticsearch.entitlement.runtime.api to org.elasticsearch.entitlement.agent;
+}

libs/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/api/EntitlementChecks.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.runtime.api;
+
+public class EntitlementChecks {
+    static boolean isAgentBooted = false;
+
+    public static void setAgentBooted() {
+        isAgentBooted = true;
+    }
+
+    public static boolean isAgentBooted() {
+        return isAgentBooted;
+    }
+}