Merge branch 'main' into wait-on-security-index

docs/changelog/111684.yaml

@@ -0,0 +1,5 @@
+pr: 111684
+summary: Write downloaded model parts async
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112850.yaml

@@ -0,0 +1,5 @@
+pr: 112850
+summary: Fix synthetic source field names for multi-fields
+area: Mapping
+type: bug
+issues: []

docs/reference/query-dsl/knn-query.asciidoc

@@ -241,7 +241,7 @@ to <<nested-knn-search, top level nested kNN search>>:
 * kNN search over nested dense_vectors diversifies the top results over
 the top-level document
 * `filter`  over the top-level document metadata is supported and acts as a
-post-filter
+pre-filter
 * `filter` over `nested` field metadata is not supported
 
 A sample query can look like below:

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -433,7 +433,7 @@ public MatchOnlyTextFieldType fieldType() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        var loader = new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName()) {
+        var loader = new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), fieldType().name(), leafName()) {
             @Override
             protected void write(XContentBuilder b, Object value) throws IOException {
                 b.value((String) value);

plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java

@@ -582,7 +582,7 @@ protected void write(XContentBuilder b, Object value) throws IOException {
 
         var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this);
         if (kwd != null) {
-            return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(leafName()));
+            return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(fullPath(), leafName()));
         }
 
         return super.syntheticSourceSupport();

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java

@@ -527,6 +527,9 @@ public String[] resolveNodes(String... nodes) {
      * Returns the changes comparing this nodes to the provided nodes.
      */
     public Delta delta(DiscoveryNodes other) {
+        if (this == other) {
+            return new Delta(this.masterNode, this.masterNode, localNodeId, List.of(), List.of());
+        }
         final List<DiscoveryNode> removed = new ArrayList<>();
         final List<DiscoveryNode> added = new ArrayList<>();
         for (DiscoveryNode node : other) {

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -1037,13 +1037,13 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
         }
 
         if (fieldType.stored() || hasDocValues) {
-            return new SyntheticSourceSupport.Native(syntheticFieldLoader(leafName()));
+            return new SyntheticSourceSupport.Native(syntheticFieldLoader(fullPath(), leafName()));
         }
 
         return super.syntheticSourceSupport();
     }
 
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String simpleName) {
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) {
         assert fieldType.stored() || hasDocValues;
 
         var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
@@ -1081,6 +1081,6 @@ protected void writeValue(Object value, XContentBuilder b) throws IOException {
             });
         }
 
-        return new CompositeSyntheticFieldLoader(simpleName, fullPath(), layers);
+        return new CompositeSyntheticFieldLoader(leafFieldName, fullFieldName, layers);
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java

@@ -19,19 +19,25 @@
 import static java.util.Collections.emptyList;
 
 public abstract class StringStoredFieldFieldLoader implements SourceLoader.SyntheticFieldLoader {
-    private final String name;
+    private final String storedFieldLoaderName;
+    private final String fullName;
     private final String simpleName;
 
     private List<Object> values = emptyList();
 
-    public StringStoredFieldFieldLoader(String name, String simpleName) {
-        this.name = name;
+    public StringStoredFieldFieldLoader(String fullName, String simpleName) {
+        this(fullName, fullName, simpleName);
+    }
+
+    public StringStoredFieldFieldLoader(String storedFieldLoaderName, String fullName, String simpleName) {
+        this.storedFieldLoaderName = storedFieldLoaderName;
+        this.fullName = fullName;
         this.simpleName = simpleName;
     }
 
     @Override
     public final Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-        return Stream.of(Map.entry(name, newValues -> values = newValues));
+        return Stream.of(Map.entry(storedFieldLoaderName, newValues -> values = newValues));
     }
 
     @Override
@@ -72,6 +78,6 @@ public final DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLe
 
     @Override
     public String fieldName() {
-        return name;
+        return fullName;
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -1462,7 +1462,7 @@ protected void write(XContentBuilder b, Object value) throws IOException {
 
         var kwd = SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this);
         if (kwd != null) {
-            return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(leafName()));
+            return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(fullPath(), leafName()));
         }
 
         return super.syntheticSourceSupport();

x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/MachineLearningPackageLoader.java

@@ -15,12 +15,17 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ExecutorBuilder;
+import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.xpack.core.ml.packageloader.action.GetTrainedModelPackageConfigAction;
 import org.elasticsearch.xpack.core.ml.packageloader.action.LoadTrainedModelPackageAction;
 import org.elasticsearch.xpack.ml.packageloader.action.ModelDownloadTask;
+import org.elasticsearch.xpack.ml.packageloader.action.ModelImporter;
 import org.elasticsearch.xpack.ml.packageloader.action.TransportGetTrainedModelPackageConfigAction;
 import org.elasticsearch.xpack.ml.packageloader.action.TransportLoadTrainedModelPackage;
 
@@ -44,16 +49,15 @@ public class MachineLearningPackageLoader extends Plugin implements ActionPlugin
         Setting.Property.Dynamic
     );
 
-    // re-using thread pool setup by the ml plugin
-    public static final String UTILITY_THREAD_POOL_NAME = "ml_utility";
-
     // This link will be invalid for serverless, but serverless will never be
     // air-gapped, so this message should never be needed.
     private static final String MODEL_REPOSITORY_DOCUMENTATION_LINK = format(
         "https://www.elastic.co/guide/en/machine-learning/%s/ml-nlp-elser.html#air-gapped-install",
         Build.current().version().replaceFirst("^(\\d+\\.\\d+).*", "$1")
     );
 
+    public static final String MODEL_DOWNLOAD_THREADPOOL_NAME = "model_download";
+
     public MachineLearningPackageLoader() {}
 
     @Override
@@ -81,6 +85,24 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         );
     }
 
+    @Override
+    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
+        return List.of(modelDownloadExecutor(settings));
+    }
+
+    public static FixedExecutorBuilder modelDownloadExecutor(Settings settings) {
+        // Threadpool with a fixed number of threads for
+        // downloading the model definition files
+        return new FixedExecutorBuilder(
+            settings,
+            MODEL_DOWNLOAD_THREADPOOL_NAME,
+            ModelImporter.NUMBER_OF_STREAMS,
+            -1, // unbounded queue size
+            "xpack.ml.model_download_thread_pool",
+            EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+        );
+    }
+
     @Override
     public List<BootstrapCheck> getBootstrapChecks() {
         return List.of(new BootstrapCheck() {