Enable deprecation log indexing by default

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -71,9 +71,11 @@ PUT /_data_stream/new-data-stream
 
 [source,console]
 ----
-DELETE /_data_stream/*
+DELETE /_data_stream/my-data-stream
+DELETE /_data_stream/new-data-stream
 
-DELETE /_index_template/*
+DELETE /_index_template/new-data-stream-template
+DELETE /_index_template/my-data-stream-template
 
 DELETE /_ilm/policy/my-data-stream-policy
 ----

docs/reference/setup/logging-config.asciidoc

@@ -234,6 +234,17 @@ The user ID is included in the `X-Opaque-ID` field in deprecation JSON logs.
 ---------------------------
 // NOTCONSOLE
 
+Deprecation logs are indexed into `.logs-deprecation.elasticsearch-default` data stream by default.
+You can disable deprecation log indexing by changing a
+`cluster.deprecation_indexing.enabled` setting to false.
+
+==== Deprecation logs throttling
+Deprecation logs are deduplicated based on a deprecated feature key
+and x-opaque-id so that if a feature is repeatedly used, it will not overload the deprecation logs.
+This applies to both indexed deprecation logs and logs emitted to log files.
+You can disable the use of `x-opaque-id` in throttling by changing
+`cluster.deprecation_indexing.x_opaque_id_used.enabled` to false
+
 [discrete]
 [[json-logging]]
 === JSON log format

server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java

@@ -31,6 +31,8 @@
 @Plugin(name = "RateLimitingFilter", category = Node.CATEGORY, elementType = Filter.ELEMENT_TYPE)
 public class RateLimitingFilter extends AbstractFilter {
 
+    private volatile boolean useXOpaqueId = true;
+
     private final Set<String> lruKeyCache = Collections.newSetFromMap(Collections.synchronizedMap(new LinkedHashMap<>() {
         @Override
         protected boolean removeEldestEntry(final Map.Entry<String, Boolean> eldest) {
@@ -57,16 +59,23 @@ public Result filter(Message message) {
         if (message instanceof ESLogMessage) {
             final ESLogMessage esLogMessage = (ESLogMessage) message;
 
-            String xOpaqueId = esLogMessage.get(X_OPAQUE_ID_FIELD_NAME);
-            final String key = esLogMessage.get(KEY_FIELD_NAME);
-
-            return lruKeyCache.add(xOpaqueId + key) ? Result.ACCEPT : Result.DENY;
+            final String key = getKey(esLogMessage);
+            return lruKeyCache.add(key) ? Result.ACCEPT : Result.DENY;
 
         } else {
             return Result.NEUTRAL;
         }
     }
 
+    private String getKey(ESLogMessage esLogMessage) {
+        final String key = esLogMessage.get(KEY_FIELD_NAME);
+        if (useXOpaqueId) {
+            String xOpaqueId = esLogMessage.get(X_OPAQUE_ID_FIELD_NAME);
+            return xOpaqueId + key;
+        }
+        return key;
+    }
+
     @Override
     public Result filter(LogEvent event) {
         return filter(event.getMessage());
@@ -84,4 +93,8 @@ public static RateLimitingFilter createFilter(
     ) {
         return new RateLimitingFilter(match, mismatch);
     }
+
+    public void setUseXOpaqueId(boolean useXOpaqueId) {
+        this.useXOpaqueId = useXOpaqueId;
+    }
 }

server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java

@@ -147,4 +147,26 @@ public void testFilterCanBeReset() {
         // Third time, it is allowed again
         assertThat(filter.filter(message), equalTo(Result.ACCEPT));
     }
+
+    public void testMessagesXOpaqueIsIgnoredWhenDisabled() {
+        RateLimitingFilter filter = new RateLimitingFilter();
+        filter.setUseXOpaqueId(false);
+        filter.start();
+
+        // Should NOT be rate-limited because it's not in the cache
+        Message message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 0", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.ACCEPT));
+
+        // Should  be rate-limited because it was just added to the cache
+        message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 0", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.DENY));
+
+        // Should be rate-limited because X-Opaque-Id is not used
+        message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 1", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.DENY));
+
+        // Should NOT be rate-limited because "key 1" it not in the cache
+        message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 1", "opaque-id 1", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.ACCEPT));
+    }
 }

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -12,6 +12,7 @@
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.http.HttpHost;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TotalHits;
@@ -69,11 +70,9 @@
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkModule;
@@ -84,7 +83,6 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -94,6 +92,9 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
@@ -171,8 +172,8 @@
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.core.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.util.CollectionUtils.eagerPartition;
+import static org.elasticsearch.core.TimeValue.timeValueMillis;
 import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING;
 import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING;
 import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING;
@@ -571,7 +572,7 @@ private void afterInternal(boolean afterClass) throws Exception {
      * @return An exclude set of index templates that will not be removed in between tests.
      */
     protected Set<String> excludeTemplates() {
-        return Collections.emptySet();
+        return Set.of(".deprecation-indexing-template");
     }
 
     protected void beforeIndexDeletion() throws Exception {

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -551,7 +551,7 @@ protected boolean preserveILMPoliciesUponCompletion() {
      */
     protected Set<String> preserveILMPolicyIds() {
         return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy",
-            "watch-history-ilm-policy", "ml-size-based-ilm-policy", "logs", "metrics");
+            "watch-history-ilm-policy", "ml-size-based-ilm-policy", "logs", "metrics", ".deprecation-indexing-ilm-policy");
     }
 
     /**
@@ -782,7 +782,7 @@ protected static void wipeAllIndices() throws IOException {
         boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0);
         try {
             //remove all indices except ilm history which can pop up after deleting all data streams but shouldn't interfere
-            final Request deleteRequest = new Request("DELETE", "*,-.ds-ilm-history-*");
+            final Request deleteRequest = new Request("DELETE", "*,-.ds-ilm-history-*,-.ds-.logs-deprecation.elasticsearch-default-*");
             deleteRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : ""));
             RequestOptions allowSystemIndexAccessWarningOptions = RequestOptions.DEFAULT.toBuilder()
                 .setWarningsHandler(warnings -> {

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json

@@ -2,6 +2,7 @@
   "template": {
     "settings": {
       "index": {
+        "hidden" : true,
         "lifecycle": {
           "name": ".deprecation-indexing-ilm-policy"
         },