Merge branch 'main' into maximum-timestamp-in-get-data-streams

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -19,11 +19,14 @@ import java.time.LocalDateTime
 develocity {
 
   buildScan {
-    URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null
-    String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null
+
+    def onCI = System.getenv('CI') ? Boolean.parseBoolean(System.getenv('CI')) : false
+
+    // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
+    uploadInBackground = onCI == false
 
     // Automatically publish scans from Elasticsearch CI
-    if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
+    if (onCI) {
       publishing.onlyIf { true }
       server = 'https://gradle-enterprise.elastic.co'
     } else if( server.isPresent() == false) {
@@ -38,73 +41,9 @@ develocity {
       if (BuildParams.inFipsJvm) {
         tag 'FIPS'
       }
-
-      // Jenkins-specific build scan metadata
-      if (jenkinsUrl) {
-        // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
-        uploadInBackground = false
-
-        String buildNumber = System.getenv('BUILD_NUMBER')
-        String buildUrl = System.getenv('BUILD_URL')
-        String jobName = System.getenv('JOB_NAME')
-        String nodeName = System.getenv('NODE_NAME')
-        String jobBranch = System.getenv('ghprbTargetBranch') ?: System.getenv('JOB_BRANCH')
-
-        // Link to Jenkins worker logs and system metrics
-        if (nodeName) {
-          link 'System logs', "https://ci-stats.elastic.co/app/infra#/logs?&logFilter=(expression:'host.name:${nodeName}',kind:kuery)"
-          buildFinished {
-            link 'System metrics', "https://ci-stats.elastic.co/app/metrics/detail/host/${nodeName}"
-          }
-        }
-
-        // Parse job name in the case of matrix builds
-        // Matrix job names come in the form of "base-job-name/matrix_param1=value1,matrix_param2=value2"
-        def splitJobName = jobName.split('/')
-        if (splitJobName.length > 1 && splitJobName.last() ==~ /^([a-zA-Z0-9_\-]+=[a-zA-Z0-9_\-&\.]+,?)+$/) {
-          def baseJobName = splitJobName.dropRight(1).join('/')
-          tag baseJobName
-          tag splitJobName.last()
-          value 'Job Name', baseJobName
-          def matrixParams = splitJobName.last().split(',')
-          matrixParams.collect { it.split('=') }.each { param ->
-            value "MATRIX_${param[0].toUpperCase()}", param[1]
-          }
-        } else {
-          tag jobName
-          value 'Job Name', jobName
-        }
-
-        tag 'CI'
-        link 'CI Build', buildUrl
-        link 'GCP Upload',
-          "https://console.cloud.google.com/storage/browser/_details/elasticsearch-ci-artifacts/jobs/${URLEncoder.encode(jobName, "UTF-8")}/build/${buildNumber}.tar.bz2"
-        value 'Job Number', buildNumber
-        if (jobBranch) {
-          tag jobBranch
-          value 'Git Branch', jobBranch
-        }
-
-        System.getenv().getOrDefault('NODE_LABELS', '').split(' ').each {
-          value 'Jenkins Worker Label', it
-        }
-
-        // Add SCM information
-        def isPrBuild = System.getenv('ROOT_BUILD_CAUSE_GHPRBCAUSE') != null
-        if (isPrBuild) {
-          value 'Git Commit ID', System.getenv('ghprbActualCommit')
-          tag "pr/${System.getenv('ghprbPullId')}"
-          tag 'pull-request'
-          link 'Source', "https://github.com/elastic/elasticsearch/tree/${System.getenv('ghprbActualCommit')}"
-          link 'Pull Request', System.getenv('ghprbPullLink')
-        } else {
-          value 'Git Commit ID', BuildParams.gitRevision
-          link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}"
-        }
-      } else if (buildKiteUrl) { //Buildkite-specific build scan metadata
-        // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
-        uploadInBackground = false
-
+      println "onCI = $onCI"
+      if (onCI) { //Buildkite-specific build scan metadata
+        String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL')
         def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH')
         def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/
         def repository = repoMatcher.matches() ? repoMatcher.group(2) : "<unknown>"

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -32,6 +32,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * The main CLI for running Elasticsearch.
@@ -44,6 +45,8 @@ class ServerCli extends EnvironmentAwareCommand {
     private final OptionSpecBuilder quietOption;
     private final OptionSpec<String> enrollmentTokenOption;
 
+    // flag for indicating shutdown has begun. we use an AtomicBoolean to double as a synchronization object
+    private final AtomicBoolean shuttingDown = new AtomicBoolean(false);
     private volatile ServerProcess server;
 
     // visible for testing
@@ -98,7 +101,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
             syncPlugins(terminal, env, processInfo);
 
             ServerArgs args = createArgs(options, env, secrets, processInfo);
-            this.server = startServer(terminal, processInfo, args);
+            synchronized (shuttingDown) {
+                // if we are shutting down there is no reason to start the server
+                if (shuttingDown.get()) {
+                    terminal.println("CLI is shutting down, skipping starting server process");
+                    return;
+                }
+                this.server = startServer(terminal, processInfo, args);
+            }
         }
 
         if (options.has(daemonizeOption)) {
@@ -233,8 +243,11 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings
 
     @Override
     public void close() throws IOException {
-        if (server != null) {
-            server.stop();
+        synchronized (shuttingDown) {
+            shuttingDown.set(true);
+            if (server != null) {
+                server.stop();
+            }
         }
     }
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java

@@ -36,6 +36,8 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -50,6 +52,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.matchesRegex;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class ServerCliTests extends CommandTestCase {
 
@@ -383,6 +386,52 @@ public void testSecureSettingsLoaderWithNullPassword() throws Exception {
         assertEquals("", loader.password);
     }
 
+    public void testProcessCreationRace() throws Exception {
+        for (int i = 0; i < 10; ++i) {
+            CyclicBarrier raceStart = new CyclicBarrier(2);
+            TestServerCli cli = new TestServerCli() {
+                @Override
+                void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception {
+                    super.syncPlugins(terminal, env, processInfo);
+                    raceStart.await();
+                }
+
+                @Override
+                public void close() throws IOException {
+                    try {
+                        raceStart.await();
+                    } catch (InterruptedException ie) {
+                        Thread.currentThread().interrupt();
+                        throw new AssertionError(ie);
+                    } catch (BrokenBarrierException e) {
+                        throw new AssertionError(e);
+                    }
+                    super.close();
+                }
+            };
+            Thread closeThread = new Thread(() -> {
+                try {
+                    cli.close();
+                } catch (IOException e) {
+                    throw new AssertionError(e);
+                }
+            });
+            closeThread.start();
+            cli.main(new String[] {}, terminal, new ProcessInfo(sysprops, envVars, esHomeDir));
+            closeThread.join();
+
+            if (cli.getServer() == null) {
+                // close won the race, so server should never have been started
+                assertThat(cli.startServerCalled, is(false));
+            } else {
+                // creation won the race, so check we correctly waited on it and stopped
+                assertThat(cli.getServer(), sameInstance(mockServer));
+                assertThat(mockServer.waitForCalled, is(true));
+                assertThat(mockServer.stopCalled, is(true));
+            }
+        }
+    }
+
     private MockSecureSettingsLoader loadWithMockSecureSettingsLoader() throws Exception {
         var loader = new MockSecureSettingsLoader();
         this.mockSecureSettingsLoader = loader;
@@ -465,9 +514,9 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
     }
 
     private class MockServerProcess extends ServerProcess {
-        boolean detachCalled = false;
-        boolean waitForCalled = false;
-        boolean stopCalled = false;
+        volatile boolean detachCalled = false;
+        volatile boolean waitForCalled = false;
+        volatile boolean stopCalled = false;
 
         MockServerProcess() {
             super(null, null);
@@ -505,6 +554,8 @@ void reset() {
     }
 
     private class TestServerCli extends ServerCli {
+        boolean startServerCalled = false;
+
         @Override
         protected Command loadTool(String toolname, String libs) {
             if (toolname.equals("auto-configure-node")) {
@@ -551,20 +602,21 @@ protected SecureSettingsLoader secureSettingsLoader(Environment env) {
 
             return new KeystoreSecureSettingsLoader();
         }
+
+        @Override
+        protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception {
+            startServerCalled = true;
+            if (argsValidator != null) {
+                argsValidator.accept(args);
+            }
+            mockServer.reset();
+            return mockServer;
+        }
     }
 
     @Override
     protected Command newCommand() {
-        return new TestServerCli() {
-            @Override
-            protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) {
-                if (argsValidator != null) {
-                    argsValidator.accept(args);
-                }
-                mockServer.reset();
-                return mockServer;
-            }
-        };
+        return new TestServerCli();
     }
 
     static class MockSecureSettingsLoader implements SecureSettingsLoader {

docs/changelog/111818.yaml

@@ -0,0 +1,5 @@
+pr: 111818
+summary: Add tier preference to security index settings allowlist
+area: Security
+type: enhancement
+issues: []

docs/changelog/111932.yaml

@@ -0,0 +1,6 @@
+pr: 111932
+summary: Fix union-types where one index is missing the field
+area: ES|QL
+type: bug
+issues:
+ - 111912

docs/changelog/112135.yaml

@@ -0,0 +1,4 @@
+pr: 112135 
+summary: Fix the bug where the run() function of ExecutableInferenceRequest throws an exception when get inferenceEntityId.
+area: Inference 
+type: bug

docs/changelog/112151.yaml

@@ -0,0 +1,5 @@
+pr: 112151
+summary: Store original source for keywords using a normalizer
+area: Logs
+type: enhancement
+issues: []

docs/changelog/112260.yaml

@@ -0,0 +1,6 @@
+pr: 112260
+summary: Fix DLS over Runtime Fields
+area: "Authorization"
+type: bug
+issues:
+ - 111637

docs/changelog/112270.yaml

@@ -0,0 +1,5 @@
+pr: 112270
+summary: Support sparse embedding models in the elasticsearch inference service
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112273.yaml

@@ -0,0 +1,5 @@
+pr: 111181
+summary: "[Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API"
+area: Machine Learning
+type: enhancement
+issues: [ ]

docs/changelog/112320.yaml

@@ -0,0 +1,5 @@
+pr: 112320
+summary: Upgrade xcontent to Jackson 2.17.2
+area: Infra/Core
+type: upgrade
+issues: []

docs/changelog/112341.yaml

@@ -0,0 +1,5 @@
+pr: 112341
+summary: Fix DLS using runtime fields and synthetic source
+area: Authorization
+type: bug
+issues: []

docs/reference/inference/inference-apis.asciidoc

@@ -39,6 +39,7 @@ include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
 include::put-inference.asciidoc[]
+include::service-alibabacloud-ai-search.asciidoc[]
 include::service-amazon-bedrock.asciidoc[]
 include::service-anthropic.asciidoc[]
 include::service-azure-ai-studio.asciidoc[]

docs/reference/inference/put-inference.asciidoc

@@ -39,6 +39,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a
 
 The following services are available through the {infer} API, click the links to review the configuration details of the services:
 
+* <<infer-service-alibabacloud-ai-search,AlibabaCloud AI Search>>
 * <<infer-service-amazon-bedrock,Amazon Bedrock>>
 * <<infer-service-anthropic,Anthropic>>
 * <<infer-service-azure-ai-studio,Azure AI Studio>>