Deprecation logs indexing is enabled by default. Backport(#78991) (#7…

…9035)

Changing the default for deprecation log indexing to be true.
This commit also overrides this default to tests where a deprecation
data stream would interfere - because it uses index template, it would
not be possible to delete with _index_template/*.
The overrides should be removed when #78850 is done.

closes #76292
backport #78991

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.gradle.Jdk;
 import org.elasticsearch.gradle.PropertyNormalization;
 import org.elasticsearch.gradle.ReaperService;
+import org.elasticsearch.gradle.Version;
 import org.gradle.api.Named;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
@@ -332,6 +333,11 @@ private void commonNodeConfig() {
         }
         ElasticsearchNode firstNode = null;
         for (ElasticsearchNode node : nodes) {
+            if (node.getTestDistribution().equals(TestDistribution.DEFAULT)) {
+                if (node.getVersion().onOrAfter("7.16.0")) {
+                    node.defaultConfig.put("cluster.deprecation_indexing.enabled", "false");
+                }
+            }
             // Can only configure master nodes if we have node names defined
             if (nodeNames != null) {
                 commonNodeConfig(node, nodeNames, firstNode);
@@ -403,6 +409,13 @@ public void nextNodeToNextVersion() {
         node.goToNextVersion();
         commonNodeConfig(node, null, null);
         // We need to translate these settings there as there's no support to do per version config for testclusters yet
+
+        if (node.getTestDistribution().equals(TestDistribution.DEFAULT)) {
+            if (node.getVersion().onOrAfter("7.16.0")) {
+                node.settings.put("cluster.deprecation_indexing.enabled", "false");
+            }
+        }
+
         if (node.getVersion().onOrAfter("7.0.0")) {
             if (node.settings.containsKey("xpack.security.authc.realms.file1.type")) {
                 node.settings.remove("xpack.security.authc.realms.file1.type");

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -104,7 +104,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS;
     private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
         "path.repo",
-        "discovery.seed_providers"
+        "discovery.seed_providers",
+        "cluster.deprecation_indexing.enabled"
 
     );
 

qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java

@@ -84,6 +84,17 @@ private void assertExit(Result result, String script, int exitCode) {
                     + "}"
             );
             logger.error(logs.stdout);
+            Result deprecationLogs = sh.run(
+                "$files = Get-ChildItem \""
+                    + installation.logs
+                    + "\\elasticsearch_deprecation.json\"; "
+                    + "Write-Output $files; "
+                    + "foreach ($file in $files) {"
+                    + "    Write-Output \"$file\"; "
+                    + "    Get-Content \"$file\" "
+                    + "}"
+            );
+            logger.error(deprecationLogs.stdout);
             fail();
         } else {
             logger.info("\nscript: " + script + "\nstdout: " + result.stdout + "\nstderr: " + result.stderr);

qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java

@@ -107,7 +107,9 @@ public static Installation installArchive(Shell sh, Distribution distribution, P
 
         Installation installation = Installation.ofArchive(sh, distribution, fullInstallPath);
         ServerUtils.disableGeoIpDownloader(installation);
-
+        if (Platforms.WINDOWS) {
+            ServerUtils.disableDeprecationLogIndexing(installation);
+        }
         return installation;
     }
 

qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -226,12 +226,12 @@ public static void waitForElasticsearch(String status, String index, Installatio
 
     public static void runElasticsearchTests() throws Exception {
         makeRequest(
-            Request.Post("http://localhost:9200/library/book/1?refresh=true&pretty")
+            Request.Post("http://localhost:9200/library/_doc/1?refresh=true&pretty")
                 .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON)
         );
 
         makeRequest(
-            Request.Post("http://localhost:9200/library/book/2?refresh=true&pretty")
+            Request.Post("http://localhost:9200/library/_doc/2?refresh=true&pretty")
                 .bodyString("{ \"title\": \"Book #2\", \"pages\": 456 }", ContentType.APPLICATION_JSON)
         );
 
@@ -266,6 +266,12 @@ public static void disableGeoIpDownloader(Installation installation) throws IOEx
         }
     }
 
+    public static void disableDeprecationLogIndexing(Installation installation) throws IOException {
+        List<String> yaml = Collections.singletonList("cluster.deprecation_indexing.enabled: false");
+        Path yml = installation.config("elasticsearch.yml");
+        Files.write(yml, yaml, CREATE, APPEND);
+    }
+
     public static void enableGeoIpDownloader(Installation installation) throws IOException {
         Path yml = installation.config("elasticsearch.yml");
         List<String> lines;

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json

@@ -3,6 +3,7 @@
     "settings": {
       "index": {
         "hidden" : true,
+        "auto_expand_replicas" : "0-1",
         "lifecycle": {
           "name": ".deprecation-indexing-ilm-policy"
         },

x-pack/plugin/deprecation/qa/rest/build.gradle

@@ -26,6 +26,7 @@ restResources {
 
 testClusters.configureEach {
   testDistribution = 'DEFAULT'
+  setting 'cluster.deprecation_indexing.enabled', 'true'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 }

x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java

@@ -65,16 +65,15 @@ public class DeprecationHttpIT extends ESRestTestCase {
 
     @Before
     public void assertIndexingIsEnabled() throws Exception {
-        configureWriteDeprecationLogsToIndex(true);
 
         // make sure the deprecation logs indexing is enabled
         Response response = client().performRequest(new Request("GET", "/_cluster/settings?include_defaults=true&flat_settings=true"));
         assertOK(response);
         ObjectMapper mapper = new ObjectMapper();
         final JsonNode jsonNode = mapper.readTree(response.getEntity().getContent());
 
-        final boolean transientValue = jsonNode.at("/transient/cluster.deprecation_indexing.enabled").asBoolean();
-        assertTrue(transientValue);
+        final boolean defaultValue = jsonNode.at("/defaults/cluster.deprecation_indexing.enabled").asBoolean();
+        assertTrue(defaultValue);
 
         // assert index does not exist, which will prevent previous tests to interfere
         assertBusy(() -> {
@@ -87,12 +86,22 @@ public void assertIndexingIsEnabled() throws Exception {
             }
             List<Map<String, Object>> documents = getIndexedDeprecations();
             logger.warn(documents);
+            // if data stream is still present, that means that previous test (could be different class) created a deprecation
+            // hence resetting again
+            resetDeprecationIndexAndCache();
             fail("Index should be removed on startup");
         }, 30, TimeUnit.SECONDS);
     }
 
     @After
     public void cleanUp() throws Exception {
+        resetDeprecationIndexAndCache();
+
+        // switch logging setting to default
+        configureWriteDeprecationLogsToIndex(null);
+    }
+
+    private void resetDeprecationIndexAndCache() throws Exception {
         // making sure the deprecation indexing cache is reset and index is deleted
         assertBusy(() -> {
             try {
@@ -103,9 +112,6 @@ public void cleanUp() throws Exception {
             }
 
         }, 30, TimeUnit.SECONDS);
-
-        // switch logging setting to default
-        configureWriteDeprecationLogsToIndex(null);
     }
 
     /**
@@ -348,8 +354,6 @@ public void testDeprecationRouteThrottling() throws Exception {
     }
 
     public void testDisableDeprecationLogIndexing() throws Exception {
-
-        configureWriteDeprecationLogsToIndex(true);
         final Request deprecatedRequest = deprecatedRequest("GET", "xOpaqueId-testDisableDeprecationLogIndexing");
         assertOK(client().performRequest(deprecatedRequest));
         configureWriteDeprecationLogsToIndex(false);

x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/MlDeprecationIT.java

@@ -10,7 +10,9 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.WarningsHandler;
@@ -27,7 +29,9 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentType;
+import org.junit.After;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 
@@ -47,6 +51,12 @@ private static class HLRC extends RestHighLevelClient {
         }
     }
 
+    @After
+    public void resetFeatures() throws IOException {
+        Response response = adminClient().performRequest(new Request("POST", "/_features/_reset"));
+        assertOK(response);
+    }
+
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
@@ -104,7 +114,6 @@ public void testMlDeprecationChecks() throws Exception {
             containsString("model snapshot [1] for job [deprecation_check_job] needs to be deleted or upgraded")
         );
         assertThat(response.getMlSettingsIssues().get(0).getMeta(), equalTo(Map.of("job_id", jobId, "snapshot_id", "1")));
-        hlrc.close();
     }
 
 }

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java

@@ -49,7 +49,7 @@ public class Deprecation extends Plugin implements ActionPlugin {
 
     public static final Setting<Boolean> WRITE_DEPRECATION_LOGS_TO_INDEX = Setting.boolSetting(
         "cluster.deprecation_indexing.enabled",
-        false,
+        true,
         Setting.Property.NodeScope,
         Setting.Property.Dynamic
     );
@@ -99,12 +99,12 @@ public Collection<Object> createComponents(
 
         final RateLimitingFilter rateLimitingFilterForIndexing = new RateLimitingFilter();
         // enable on start.
-        rateLimitingFilterForIndexing.setUseXOpaqueId(USE_X_OPAQUE_ID_IN_FILTERING.getDefault(environment.settings()));
+        rateLimitingFilterForIndexing.setUseXOpaqueId(USE_X_OPAQUE_ID_IN_FILTERING.get(environment.settings()));
 
         final DeprecationIndexingComponent component = new DeprecationIndexingComponent(client,
             environment.settings(),
             rateLimitingFilterForIndexing,
-            WRITE_DEPRECATION_LOGS_TO_INDEX.getDefault(environment.settings()) //pass the default on startup
+            WRITE_DEPRECATION_LOGS_TO_INDEX.get(environment.settings()) //pass the default on startup
         );
 
         clusterService.getClusterSettings().addSettingsUpdateConsumer(USE_X_OPAQUE_ID_IN_FILTERING,

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java

@@ -30,7 +30,7 @@
 import org.elasticsearch.xpack.core.ClientHelper;
 
 import java.util.Arrays;
-import java.util.Map;
+import java.util.List;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -137,9 +137,10 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon
             }
 
             if (response.hasFailures()) {
-                Map<String, String> failures = Arrays.stream(response.getItems())
+                List<String> failures = Arrays.stream(response.getItems())
                     .filter(BulkItemResponse::isFailed)
-                    .collect(Collectors.toMap(BulkItemResponse::getId, BulkItemResponse::getFailureMessage));
+                    .map(r -> r.getId() + " " + r.getFailureMessage() )
+                    .collect(Collectors.toList());
                 logger.error("Bulk write of deprecation logs encountered some failures: [{}]", failures);
             }
         }