Merge branch 'master' into feature/data_stream_support_routing

* master: (185 commits)
  Implement get and containsKey in terms of the wrapped innerMap (elastic#77965)
  Adjust Lucene version and enable BWC tests (elastic#77933)
  Disable BWC to upgrade to Lucene-8.10-snapshot
  Reenable MlDistributedFailureIT
  [DOCS] Fix typo for `script.painless.regex.enabled` setting value (elastic#77853)
  Upgrade to Lucene-8.10.0-snapshot-bf2fcb53079 (elastic#77801)
  [DOCS] Fix ESS install lead-in (elastic#77887)
  Resolve thirdparty gradle plugin artifacts from mavencentral (elastic#77865)
  Reduce the number of times that `LifecycleExecutionState` is parsed when running a policy. (elastic#77863)
  Utility methods to add and remove backing indices from data streams (elastic#77778)
  Use Objects.equals() instead of == to compare strings (elastic#77840)
  [ML] prefer least allocated model when a new node is added to the cluster (elastic#77756)
  Deprecate ignore_throttled parameter (elastic#77479)
  Improve LifecycleExecutionState parsing. (elastic#77855)
  [DOCS] Removes deprecated word from HLRC title. (elastic#77851)
  Remove legacy geo code from AggregationResultUtils (elastic#77702)
  Adjust SearchableSnapshotsBlobStoreCacheIntegTests.testBlobStoreCache (elastic#77758)
  Laxify SecureSM to allow creation of the JDK's innocuous threads (elastic#77789)
  [Test] Reduce concurrency when testing creation of security index (elastic#75293)
  Refactor metric PipelineAggregation integration test (elastic#77548)
  ...

# Conflicts:
#	server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml

@@ -25,6 +25,7 @@
             - sles-15-packaging
             - ubuntu-18.04-packaging
             - ubuntu-20.04-packaging
+            - rocky-linux-8-packaging
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -25,6 +25,7 @@
             - "sles-15&&immutable"
             - "ubuntu-18.04&&immutable"
             - "ubuntu-20.04&&immutable"
+            - "rocky-linux-8&&immutable"
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml

@@ -44,6 +44,7 @@
             - sles-15-packaging
             - ubuntu-18.04-packaging
             - ubuntu-20.04-packaging
+            - rocky-linux-8-packaging
       - axis:
           type: user-defined
           name: PACKAGING_TASK

.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml

@@ -31,7 +31,7 @@
           properties-file: '.ci/java-versions.properties'
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
-            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml

@@ -31,7 +31,7 @@
           properties-file: '.ci/java-versions.properties'
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
-            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/java11
             JAVA15_HOME=$HOME/.java/openjdk15
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java

@@ -0,0 +1,138 @@
+package org.elasticsearch.benchmark.search.fetch.subphase;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.support.filtering.FilterPath;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.fetch.subphase.FetchSourcePhase;
+import org.elasticsearch.search.lookup.SourceLookup;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+@Fork(1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+public class FetchSourcePhaseBenchmark {
+    private BytesReference sourceBytes;
+    private FetchSourceContext fetchContext;
+    private Set<String> includesSet;
+    private Set<String> excludesSet;
+    private FilterPath[] includesFilters;
+    private FilterPath[] excludesFilters;
+
+    @Param({ "tiny", "short", "one_4k_field", "one_4m_field" })
+    private String source;
+    @Param({ "message" })
+    private String includes;
+    @Param({ "" })
+    private String excludes;
+
+    @Setup
+    public void setup() throws IOException {
+        switch (source) {
+            case "tiny":
+                sourceBytes = new BytesArray("{\"message\": \"short\"}");
+                break;
+            case "short":
+                sourceBytes = read300BytesExample();
+                break;
+            case "one_4k_field":
+                sourceBytes = buildBigExample("huge".repeat(1024));
+                break;
+            case "one_4m_field":
+                sourceBytes = buildBigExample("huge".repeat(1024 * 1024));
+                break;
+            default:
+                throw new IllegalArgumentException("Unknown source [" + source + "]");
+        }
+        fetchContext = new FetchSourceContext(
+            true,
+            Strings.splitStringByCommaToArray(includes),
+            Strings.splitStringByCommaToArray(excludes)
+        );
+        includesSet = Set.of(fetchContext.includes());
+        excludesSet = Set.of(fetchContext.excludes());
+        includesFilters = FilterPath.compile(Set.of(fetchContext.includes()));
+        excludesFilters = FilterPath.compile(Set.of(fetchContext.excludes()));
+    }
+
+    private BytesReference read300BytesExample() throws IOException {
+        return Streams.readFully(FetchSourcePhaseBenchmark.class.getResourceAsStream("300b_example.json"));
+    }
+
+    private BytesReference buildBigExample(String extraText) throws IOException {
+        String bigger = read300BytesExample().utf8ToString();
+        bigger = "{\"huge\": \"" + extraText + "\"," + bigger.substring(1);
+        return new BytesArray(bigger);
+    }
+
+    @Benchmark
+    public BytesReference filterObjects() throws IOException {
+        SourceLookup lookup = new SourceLookup();
+        lookup.setSource(sourceBytes);
+        Object value = lookup.filter(fetchContext);
+        return FetchSourcePhase.objectToBytes(value, XContentType.JSON, Math.min(1024, lookup.internalSourceRef().length()));
+    }
+
+    @Benchmark
+    public BytesReference filterXContentOnParser() throws IOException {
+        BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length()));
+        XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput);
+        try (
+            XContentParser parser = XContentType.JSON.xContent()
+                .createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    sourceBytes.streamInput(),
+                    includesFilters,
+                    excludesFilters
+                )
+        ) {
+            builder.copyCurrentStructure(parser);
+            return BytesReference.bytes(builder);
+        }
+    }
+
+    @Benchmark
+    public BytesReference filterXContentOnBuilder() throws IOException {
+        BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length()));
+        XContentBuilder builder = new XContentBuilder(
+            XContentType.JSON.xContent(),
+            streamOutput,
+            includesSet,
+            excludesSet,
+            XContentType.JSON.toParsedMediaType()
+        );
+        try (
+            XContentParser parser = XContentType.JSON.xContent()
+                .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, sourceBytes.streamInput())
+        ) {
+            builder.copyCurrentStructure(parser);
+            return BytesReference.bytes(builder);
+        }
+    }
+}

benchmarks/src/main/resources/org/elasticsearch/benchmark/search/fetch/subphase/300b_example.json

@@ -0,0 +1,20 @@
+{
+  "@timestamp": "2099-11-15T14:12:12",
+  "http": {
+    "request": {
+      "method": "get"
+    },
+    "response": {
+      "bytes": 1070000,
+      "status_code": 200
+    },
+    "version": "1.1"
+  },
+  "message": "GET /search HTTP/1.1 200 1070000",
+  "source": {
+    "ip": "192.168.0.1"
+  },
+  "user": {
+    "id": "user"
+  }
+}

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

@@ -110,7 +110,7 @@ public void setExcludes(List<String> excludes) {
      * Allowed license families for this project.
      */
     @Input
-    private List<String> approvedLicenses = new ArrayList<String>(Arrays.asList("SSPL+Elastic License", "Generated", "Vendored"));
+    private List<String> approvedLicenses = new ArrayList<String>(Arrays.asList("SSPL+Elastic License", "Generated", "Vendored", "Apache LZ4-Java"));
     /**
      * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the
      * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the
@@ -154,6 +154,8 @@ public void runRat() {
         matchers.add(subStringMatcher("BSD4 ", "Original BSD License (with advertising clause)", "All advertising materials"));
         // Apache
         matchers.add(subStringMatcher("AL   ", "Apache", "Licensed to Elasticsearch B.V. under one or more contributor"));
+        // Apache lz4-java
+        matchers.add(subStringMatcher("ALLZ4", "Apache LZ4-Java", "Copyright 2020 Adrien Grand and the lz4-java contributors"));
         // Generated resources
         matchers.add(subStringMatcher("GEN  ", "Generated", "ANTLR GENERATED CODE"));
         // Vendored Code

build-tools-internal/build.gradle

@@ -102,9 +102,9 @@ gradlePlugin {
       id = 'elasticsearch.java'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin'
     }
-    javaRestTest {
-      id = 'elasticsearch.java-rest-test'
-      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.JavaRestTestPlugin'
+    internalJavaRestTest {
+      id = 'elasticsearch.internal-java-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin'
     }
     jdkDownload {
       id = 'elasticsearch.jdk-download'

build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle

@@ -54,6 +54,7 @@ def projectPathsToExclude = [
   ':libs:elasticsearch-dissect',
   ':libs:elasticsearch-geo',
   ':libs:elasticsearch-grok',
+  ':libs:elasticsearch-lz4',
   ':libs:elasticsearch-nio',
   ':libs:elasticsearch-plugin-classloader',
   ':libs:elasticsearch-secure-sm',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java

@@ -8,6 +8,9 @@
 
 package org.elasticsearch.gradle.internal.docker;
 
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
 /**
  * The methods in this class take a shell command and wrap it in retry logic, so that our
  * Docker builds can be more robust in the face of transient errors e.g. network issues.
@@ -20,7 +23,11 @@ static String loop(String name, String command) {
     static String loop(String name, String command, int indentSize, String exitKeyword) {
         String indent = " ".repeat(indentSize);
 
-        StringBuilder commandWithRetry = new StringBuilder("for iter in {1..10}; do \n");
+        // bash understands the `{1..10}` syntax, but other shells don't e.g. the default in Alpine Linux.
+        // We therefore use an explicit sequence.
+        String retrySequence = IntStream.rangeClosed(1, 10).mapToObj(String::valueOf).collect(Collectors.joining(" "));
+
+        StringBuilder commandWithRetry = new StringBuilder("for iter in " + retrySequence + "; do \n");
         commandWithRetry.append(indent).append("  ").append(command).append(" && \n");
         commandWithRetry.append(indent).append("  exit_code=0 && break || \n");
         commandWithRetry.append(indent);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/JavaRestTestPlugin.java → build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java

@@ -21,7 +21,7 @@
 /**
  * Apply this plugin to run the Java based REST tests.
  */
-public class JavaRestTestPlugin implements Plugin<Project> {
+public class InternalJavaRestTestPlugin implements Plugin<Project> {
 
     public static final String SOURCE_SET_NAME = "javaRestTest";
 

build-tools-internal/src/main/resources/checkstyle.xml

@@ -95,6 +95,17 @@
       lines up with the directory structure. -->
     <module name="PackageDeclaration" />
 
+    <!-- Checks that a local variable or a parameter does not shadow a field that is defined in the same class. -->
+    <!-- Disabled until existing violations are fixed -->
+    <!--
+    <module name="HiddenField">
+        <property name="ignoreConstructorParameter" value="true" />
+        <property name="ignoreSetter" value="true" />
+        <property name="setterCanReturnItsClass" value="true"/>
+        <property name="ignoreFormat" value="^(threadPool)$"/>
+    </module>
+    -->
+
     <!-- We don't use Java's builtin serialization and we suppress all warning
       about it. The flip side of that coin is that we shouldn't _try_ to use
       it. We can't outright ban it with ForbiddenApis because it complain about

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.9.0
+lucene            = 8.10.0-snapshot-bf2fcb53079
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 16.0.2+7

build-tools/build.gradle

@@ -46,6 +46,10 @@ gradlePlugin {
             id = 'elasticsearch.esplugin'
             implementationClass = 'org.elasticsearch.gradle.plugin.PluginBuildPlugin'
         }
+        javaRestTest {
+            id = 'elasticsearch.java-rest-test'
+            implementationClass = 'org.elasticsearch.gradle.test.JavaRestTestPlugin'
+        }
         testclusters {
             id = 'elasticsearch.testclusters'
             implementationClass = 'org.elasticsearch.gradle.testclusters.TestClustersPlugin'

build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.test
+
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.TaskOutcome
+
+class JavaRestTestPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "declares default dependencies"() {
+        given:
+        buildFile << """
+        plugins {
+          id 'elasticsearch.java-rest-test'
+        }
+        """
+
+        when:
+        def result = gradleRunner("dependencies").build()
+        def output = normalized(result.output)
+        then:
+        output.contains(normalized("""
+javaRestTestImplementation - Implementation only dependencies for source set 'java rest test'. (n)
+/--- org.elasticsearch.test:framework:${VersionProperties.elasticsearch} (n)"""))
+    }
+
+    def "javaRestTest does nothing when there are no tests"() {
+        given:
+        buildFile << """
+        plugins {
+          id 'elasticsearch.java-rest-test'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+
+        dependencies {
+            javaRestTestImplementation "org.elasticsearch.test:framework:7.14.0"
+        }
+        """
+
+        when:
+        def result = gradleRunner("javaRestTest").build()
+        then:
+        result.task(':compileJavaRestTestJava').outcome == TaskOutcome.NO_SOURCE
+        result.task(':javaRestTest').outcome == TaskOutcome.NO_SOURCE
+    }
+
+}