Allow index filtering in field capabilities API (#57276) (#58299)

This change allows to use an `index_filter` in the
field capabilities API. Indices are filtered from
the response if the provided query rewrites to `match_none`
on every shard:

````
GET metrics-*
{
  "index_filter": {
    "bool": {
      "must": [
        "range": {
          "@timestamp": {
            "gt": "2019"
          }
        }
      }
  }
}
````

The filtering is done on a best-effort basis, it uses the can match phase
to rewrite queries to `match_none` instead of fully executing the request.
The first shard that can match the filter is used to create the field
capabilities response for the entire index.

Closes #56195

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -539,13 +539,17 @@ static Request explain(ExplainRequest explainRequest) throws IOException {
         return request;
     }
 
-    static Request fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest) {
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint(fieldCapabilitiesRequest.indices(), "_field_caps"));
+    static Request fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest) throws IOException {
+        String methodName = fieldCapabilitiesRequest.indexFilter() != null ? HttpPost.METHOD_NAME  : HttpGet.METHOD_NAME;
+        Request request = new Request(methodName, endpoint(fieldCapabilitiesRequest.indices(), "_field_caps"));
 
         Params params = new Params();
         params.withFields(fieldCapabilitiesRequest.fields());
         params.withIndicesOptions(fieldCapabilitiesRequest.indicesOptions());
         request.addParameters(params.asMap());
+        if (fieldCapabilitiesRequest.indexFilter() != null) {
+            request.setEntity(createEntity(fieldCapabilitiesRequest, REQUEST_BODY_CONTENT_TYPE));
+        }
         return request;
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -1661,7 +1661,7 @@ public void testMultiTermVectorsWithType() throws IOException {
         assertToXContentBody(mtvRequest, request.getEntity());
     }
 
-    public void testFieldCaps() {
+    public void testFieldCaps() throws IOException {
         // Create a random request.
         String[] indices = randomIndicesNames(0, 5);
         String[] fields = generateRandomStringArray(5, 10, false, false);
@@ -1699,6 +1699,48 @@ public void testFieldCaps() {
         assertNull(request.getEntity());
     }
 
+    public void testFieldCapsWithIndexFilter() throws IOException {
+        // Create a random request.
+        String[] indices = randomIndicesNames(0, 5);
+        String[] fields = generateRandomStringArray(5, 10, false, false);
+
+        FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest()
+            .indices(indices)
+            .fields(fields)
+            .indexFilter(QueryBuilders.matchAllQuery());
+
+        Map<String, String> indicesOptionsParams = new HashMap<>();
+        setRandomIndicesOptions(fieldCapabilitiesRequest::indicesOptions, fieldCapabilitiesRequest::indicesOptions, indicesOptionsParams);
+
+        Request request = RequestConverters.fieldCaps(fieldCapabilitiesRequest);
+
+        // Verify that the resulting REST request looks as expected.
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        String joinedIndices = String.join(",", indices);
+        if (!joinedIndices.isEmpty()) {
+            endpoint.add(joinedIndices);
+        }
+        endpoint.add("_field_caps");
+
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(5, request.getParameters().size());
+
+        // Note that we don't check the field param value explicitly, as field names are
+        // passed through
+        // a hash set before being added to the request, and can appear in a
+        // non-deterministic order.
+        assertThat(request.getParameters(), hasKey("fields"));
+        String[] requestFields = Strings.splitStringByCommaToArray(request.getParameters().get("fields"));
+        assertEquals(new HashSet<>(Arrays.asList(fields)), new HashSet<>(Arrays.asList(requestFields)));
+
+        for (Map.Entry<String, String> param : indicesOptionsParams.entrySet()) {
+            assertThat(request.getParameters(), hasEntry(param.getKey(), param.getValue()));
+        }
+
+        assertNotNull(request.getEntity());
+        assertToXContentBody(fieldCapabilitiesRequest, request.getEntity());
+    }
+
     public void testRankEval() throws Exception {
         RankEvalSpec spec = new RankEvalSpec(
                 Collections.singletonList(new RatedRequest("queryId", Collections.emptyList(), new SearchSourceBuilder())),

docs/reference/search/field-caps.asciidoc

@@ -12,11 +12,11 @@ GET /_field_caps?fields=rating
 [[search-field-caps-api-request]]
 ==== {api-request-title}
 
-`GET /_field_caps` 
+`GET /_field_caps`
 
-`POST /_field_caps` 
+`POST /_field_caps`
 
-`GET /<index>/_field_caps` 
+`GET /<index>/_field_caps`
 
 `POST /<index>/_field_caps`
 
@@ -25,7 +25,7 @@ GET /_field_caps?fields=rating
 ==== {api-description-title}
 
 
-The field capabilities API returns the information about the capabilities of 
+The field capabilities API returns the information about the capabilities of
 fields among multiple indices.
 
 
@@ -53,13 +53,19 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=fields]
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
 
 `include_unmapped`::
-  (Optional, boolean) If `true`, unmapped fields are included in the response. 
+  (Optional, boolean) If `true`, unmapped fields are included in the response.
   Defaults to `false`.
 
+[[search-field-caps-api-request-body]]
+==== {api-request-body-title}
+
+`index_filter`::
+(Optional,  <<query-dsl,query object>> Allows to filter indices if the provided
+query rewrites to `match_none` on every shard.
 
 [[search-field-caps-api-response-body]]
 ==== {api-response-body-title}
-  
+
 
 
 `searchable`::
@@ -69,15 +75,15 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
   Whether this field can be aggregated on all indices.
 
 `indices`::
-  The list of indices where this field has the same type, or null if all indices 
+  The list of indices where this field has the same type, or null if all indices
   have the same type for the field.
 
 `non_searchable_indices`::
-  The list of indices where this field is not searchable, or null if all indices 
+  The list of indices where this field is not searchable, or null if all indices
   have the same definition for the field.
 
 `non_aggregatable_indices`::
-  The list of indices where this field is not aggregatable, or null if all 
+  The list of indices where this field is not aggregatable, or null if all
   indices have the same definition for the field.
 
 `meta`::
@@ -100,7 +106,7 @@ GET twitter/_field_caps?fields=rating
 // TEST[setup:twitter]
 
 
-The next example API call requests information about the `rating` and the 
+The next example API call requests information about the `rating` and the
 `title` fields:
 
 [source,console]
@@ -156,7 +162,7 @@ adding a parameter called `include_unmapped` in the request:
 GET _field_caps?fields=rating,title&include_unmapped
 --------------------------------------------------
 
-In which case the response will contain an entry for each field that is present 
+In which case the response will contain an entry for each field that is present
 in some indices but not all:
 
 [source,console-result]
@@ -202,3 +208,37 @@ in some indices but not all:
 
 <1> The `rating` field is unmapped` in `index5`.
 <2> The `title` field is unmapped` in `index5`.
+
+It is also possible to filter indices with a query:
+
+[source,console]
+--------------------------------------------------
+POST twitter*/_field_caps?fields=rating
+{
+    "index_filter": {
+        "range": {
+            "@timestamp": {
+                "gte": "2018"
+             }
+        }
+    }
+}
+--------------------------------------------------
+// TEST[setup:twitter]
+
+
+In which case indices that rewrite the provided filter to `match_none` on every shard
+will be filtered from the response.
+
+--
+[IMPORTANT]
+====
+The filtering is done on a best-effort basis, it uses index statistics and mappings
+to rewrite queries to `match_none` instead of fully executing the request.
+For instance a `range` query over a `date` field can rewrite to `match_none`
+if all documents within a shard (including deleted documents) are outside
+of the provided range.
+However, not all queries can rewrite to `match_none` so this API may return
+an index even if the provided filter matches no document.
+====
+--

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml

@@ -30,15 +30,17 @@
   - match: {indices.6.name: my_remote_cluster:closed_index}
   - match: {indices.6.aliases.0: aliased_closed_index}
   - match: {indices.6.attributes.0: closed}
-  - match: {indices.7.name: my_remote_cluster:field_caps_index_1}
+  - match: {indices.7.name: my_remote_cluster:field_caps_empty_index}
   - match: {indices.7.attributes.0: open}
-  - match: {indices.8.name: my_remote_cluster:field_caps_index_3}
+  - match: {indices.8.name: my_remote_cluster:field_caps_index_1}
   - match: {indices.8.attributes.0: open}
-  - match: {indices.9.name: my_remote_cluster:single_doc_index}
+  - match: {indices.9.name: my_remote_cluster:field_caps_index_3}
   - match: {indices.9.attributes.0: open}
-  - match: {indices.10.name: my_remote_cluster:test_index}
-  - match: {indices.10.aliases.0: aliased_test_index}
+  - match: {indices.10.name: my_remote_cluster:single_doc_index}
   - match: {indices.10.attributes.0: open}
+  - match: {indices.11.name: my_remote_cluster:test_index}
+  - match: {indices.11.aliases.0: aliased_test_index}
+  - match: {indices.11.attributes.0: open}
   - match: {aliases.0.name: my_remote_cluster:aliased_closed_index}
   - match: {aliases.0.indices.0: closed_index}
   - match: {aliases.1.name: my_remote_cluster:aliased_test_index}

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml

@@ -316,7 +316,7 @@
   - do:
       search:
         rest_total_hits_as_int: true
-        index: my_remote_cluster:aliased_test_index,my_remote_cluster:field_caps_index_1
+        index: my_remote_cluster:aliased_test_index,my_remote_cluster:field_caps_empty_index
 
   - is_false: num_reduce_phases
   - match: {_clusters.total: 1}

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml

@@ -72,3 +72,54 @@
         fields: [number]
   - match: {fields.number.double.searchable:            true}
   - match: {fields.number.double.aggregatable:          true}
+
+---
+"Get field caps from remote cluster with index filter":
+  - skip:
+      version: " - 7.8.99"
+      reason: Index filter support was added in 7.9
+
+  - do:
+      indices.create:
+        index: field_caps_index_4
+        body:
+          mappings:
+            properties:
+              text:
+                type:     text
+              keyword:
+                type:     keyword
+              number:
+                type:     double
+              geo:
+                type:     geo_point
+  - do:
+      index:
+        index:  field_caps_index_4
+        body:   { created_at: "2017-01-02" }
+
+  - do:
+      indices.refresh:
+        index: [field_caps_index_4]
+
+  - do:
+      field_caps:
+        index: 'field_caps_index_4,my_remote_cluster:field_*'
+        fields: [number]
+        body: { index_filter: { range: { created_at: { lt: 2018 } } } }
+
+  - match: {indices:                                    ["field_caps_index_4","my_remote_cluster:field_caps_index_1"]}
+  - length: {fields.number:                             1}
+  - match: {fields.number.double.searchable:            true}
+  - match: {fields.number.double.aggregatable:          true}
+
+  - do:
+      field_caps:
+        index: 'field_caps_index_4,my_remote_cluster:field_*'
+        fields: [number]
+        body: { index_filter: { range: { created_at: { gt: 2019 } } } }
+
+  - match: {indices:                                    ["my_remote_cluster:field_caps_index_3"]}
+  - length: {fields.number:                             1}
+  - match: {fields.number.long.searchable:              true}
+  - match: {fields.number.long.aggregatable:            true}

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml

@@ -50,7 +50,6 @@
         body:
           settings:
             index:
-              number_of_shards: 1
               number_of_replicas: 0
           mappings:
             properties:
@@ -64,15 +63,18 @@
         body:
             - '{"index": {"_index": "single_doc_index"}}'
             - '{"f1": "remote_cluster", "sort_field": 1, "created_at" : "2016-01-01"}'
+  - do:
+        indices.create:
+          index: field_caps_empty_index
 
   - do:
         indices.create:
           index: field_caps_index_1
           body:
-              settings:
-                index.number_of_shards: 1
               mappings:
                 properties:
+                  created_at:
+                    type:     date
                   text:
                     type:     text
                   keyword:
@@ -94,10 +96,10 @@
         indices.create:
           index: field_caps_index_3
           body:
-              settings:
-                index.number_of_shards: 1
               mappings:
                 properties:
+                  created_at:
+                    type:     date
                   text:
                     type:     text
                   keyword:
@@ -158,6 +160,20 @@
            - '{"index": {"_index": "test_index"}}'
            - '{"f1": "remote_cluster", "animal": "chicken", "filter_field": 0}'
 
+  - do:
+      bulk:
+        refresh: true
+        body:
+          # Force all documents to be in the same shard (same routing)
+          - '{"index": {"_index": "field_caps_index_1", "routing": "foo"}}'
+          - '{"created_at": "2018-01-05"}'
+          - '{"index": {"_index": "field_caps_index_1", "routing": "foo"}}'
+          - '{"created_at": "2017-12-01"}'
+          - '{"index": {"_index": "field_caps_index_3"}}'
+          - '{"created_at": "2019-10-01"}'
+          - '{"index": {"_index": "field_caps_index_3"}}'
+          - '{"created_at": "2020-01-01"}'
+
   - do:
       search:
         rest_total_hits_as_int: true

rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json

@@ -59,6 +59,9 @@
         "default":false,
         "description":"Indicates whether unmapped fields should be included in the response."
       }
+    },
+    "body":{
+      "description":"An index filter specified with the Query DSL"
     }
   }
 }