Setting to disable x-opaque-id in logs throttling

docs/reference/setup/logging-config.asciidoc

@@ -234,6 +234,17 @@ The user ID is included in the `X-Opaque-ID` field in deprecation JSON logs.
 ---------------------------
 // NOTCONSOLE
 
+Deprecation logs can be indexed into `.logs-deprecation.elasticsearch-default` data stream
+`cluster.deprecation_indexing.enabled` setting is set to true.
+
+==== Deprecation logs throttling
+Deprecation logs are deduplicated based on a deprecated feature key
+and x-opaque-id so that if a feature is repeatedly used, it will not overload the deprecation logs.
+This applies to both indexed deprecation logs and logs emitted to log files.
+You can disable the use of `x-opaque-id` in throttling by changing
+`cluster.deprecation_indexing.x_opaque_id_used.enabled` to false
+See link:./server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java[RateLimitingFilter]
+
 [discrete]
 [[json-logging]]
 === JSON log format

server/src/main/java/org/elasticsearch/common/logging/RateLimitingFilter.java

@@ -28,9 +28,21 @@
 import static org.elasticsearch.common.logging.DeprecatedMessage.KEY_FIELD_NAME;
 import static org.elasticsearch.common.logging.DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME;
 
+/**
+ * A filter used for throttling deprecation logs.
+ * A throttling is based on a combined key which consists of `key` from the logged ESMessage and `x-opaque-id`
+ * passed by a user on a HTTP header.
+ * This filter works by using a lruKeyCache - a set of keys which prevents a second message with the same key to be logged.
+ * The lruKeyCache has a size limited to 128, which when breached will remove the oldest entries.
+ *
+ * It is possible to disable use of `x-opaque-id` as a key with {@link RateLimitingFilter#setUseXOpaqueId(boolean) }
+ * @see <a href="https://logging.apache.org/log4j/2.x/manual/filters.htmlf">Log4j2 Filters</a>
+ */
 @Plugin(name = "RateLimitingFilter", category = Node.CATEGORY, elementType = Filter.ELEMENT_TYPE)
 public class RateLimitingFilter extends AbstractFilter {
 
+    private volatile boolean useXOpaqueId = true;
+
     private final Set<String> lruKeyCache = Collections.newSetFromMap(Collections.synchronizedMap(new LinkedHashMap<>() {
         @Override
         protected boolean removeEldestEntry(final Map.Entry<String, Boolean> eldest) {
@@ -57,16 +69,23 @@ public Result filter(Message message) {
         if (message instanceof ESLogMessage) {
             final ESLogMessage esLogMessage = (ESLogMessage) message;
 
-            String xOpaqueId = esLogMessage.get(X_OPAQUE_ID_FIELD_NAME);
-            final String key = esLogMessage.get(KEY_FIELD_NAME);
-
-            return lruKeyCache.add(xOpaqueId + key) ? Result.ACCEPT : Result.DENY;
+            final String key = getKey(esLogMessage);
+            return lruKeyCache.add(key) ? Result.ACCEPT : Result.DENY;
 
         } else {
             return Result.NEUTRAL;
         }
     }
 
+    private String getKey(ESLogMessage esLogMessage) {
+        final String key = esLogMessage.get(KEY_FIELD_NAME);
+        if (useXOpaqueId) {
+            String xOpaqueId = esLogMessage.get(X_OPAQUE_ID_FIELD_NAME);
+            return xOpaqueId + key;
+        }
+        return key;
+    }
+
     @Override
     public Result filter(LogEvent event) {
         return filter(event.getMessage());
@@ -84,4 +103,8 @@ public static RateLimitingFilter createFilter(
     ) {
         return new RateLimitingFilter(match, mismatch);
     }
+
+    public void setUseXOpaqueId(boolean useXOpaqueId) {
+        this.useXOpaqueId = useXOpaqueId;
+    }
 }

server/src/test/java/org/elasticsearch/common/logging/RateLimitingFilterTests.java

@@ -147,4 +147,26 @@ public void testFilterCanBeReset() {
         // Third time, it is allowed again
         assertThat(filter.filter(message), equalTo(Result.ACCEPT));
     }
+
+    public void testMessagesXOpaqueIsIgnoredWhenDisabled() {
+        RateLimitingFilter filter = new RateLimitingFilter();
+        filter.setUseXOpaqueId(false);
+        filter.start();
+
+        // Should NOT be rate-limited because it's not in the cache
+        Message message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 0", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.ACCEPT));
+
+        // Should  be rate-limited because it was just added to the cache
+        message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 0", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.DENY));
+
+        // Should be rate-limited because X-Opaque-Id is not used
+        message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 0", "opaque-id 1", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.DENY));
+
+        // Should NOT be rate-limited because "key 1" it not in the cache
+        message = DeprecatedMessage.of(DeprecationCategory.OTHER, "key 1", "opaque-id 1", "msg 0");
+        assertThat(filter.filter(message), equalTo(Result.ACCEPT));
+    }
 }

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json

@@ -2,6 +2,7 @@
   "template": {
     "settings": {
       "index": {
+        "hidden" : true,
         "lifecycle": {
           "name": ".deprecation-indexing-ilm-policy"
         },