Skip shards when querying constant keyword fields

docs/changelog/96161.yaml

@@ -0,0 +1,6 @@
+pr: 96161
+summary: Skip shards when querying constant keyword fields
+area: "Search"
+type: enhancement
+issues:
+ - 95541

server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java

@@ -15,16 +15,23 @@
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.ExistsQueryBuilder;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.refresh.RefreshStats;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Arrays;
+import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Phaser;
@@ -223,4 +230,175 @@ public void testSearchIdleStats() throws InterruptedException {
         assertTrue(Arrays.stream(statsResponse.getShards()).allMatch(x -> x.getSearchIdleTime() >= searchIdleAfter));
     }
 
+    public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
+        // GIVEN
+        final String idleIndex = "test1";
+        final String activeIndex = "test2";
+        // NOTE: we need many shards because shard pre-filtering and the "can match" phase
+        // are executed only if we have enough shards.
+        int idleIndexShardsCount = 3;
+        int activeIndexShardsCount = 3;
+        createIndex(
+            idleIndex,
+            Settings.builder()
+                .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, idleIndexShardsCount)
+                .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+                .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
+                .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-10T00:00:00.000Z")
+                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-11T00:00:00.000Z")
+                .build(),
+            "doc",
+            "keyword",
+            "type=keyword",
+            "@timestamp",
+            "type=date",
+            "routing_field",
+            "type=keyword,time_series_dimension=true"
+        );
+        createIndex(
+            activeIndex,
+            Settings.builder()
+                .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, activeIndexShardsCount)
+                .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+                .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
+                .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-12T00:00:00.000Z")
+                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-13T23:59:59.999Z")
+                .build(),
+            "doc",
+            "keyword",
+            "type=keyword",
+            "@timestamp",
+            "type=date",
+            "routing_field",
+            "type=keyword,time_series_dimension=true"
+        );
+
+        assertEquals(
+            RestStatus.CREATED,
+            client().prepareIndex(idleIndex)
+                .setSource("keyword", "idle", "@timestamp", "2021-05-10T19:00:03.765Z", "routing_field", "aaa")
+                .get()
+                .status()
+        );
+        assertEquals(
+            RestStatus.CREATED,
+            client().prepareIndex(activeIndex)
+                .setSource("keyword", "active", "@timestamp", "2021-05-12T20:07:12.112Z", "routing_field", "aaa")
+                .get()
+                .status()
+        );
+        assertEquals(RestStatus.OK, client().admin().indices().prepareRefresh(idleIndex, activeIndex).get().getStatus());
+
+        waitUntil(
+            () -> Arrays.stream(client().admin().indices().prepareStats(idleIndex, activeIndex).get().getShards())
+                .allMatch(ShardStats::isSearchIdle),
+            2,
+            TimeUnit.SECONDS
+        );
+
+        final IndicesStatsResponse idleIndexStatsBefore = client().admin().indices().prepareStats("test1").get();
+        assertIdleShard(idleIndexStatsBefore);
+
+        final IndicesStatsResponse activeIndexStatsBefore = client().admin().indices().prepareStats("test2").get();
+        assertIdleShard(activeIndexStatsBefore);
+
+        // WHEN
+        final SearchResponse searchResponse = client().prepareSearch("test*")
+            .setQuery(new RangeQueryBuilder("@timestamp").from("2021-05-12T20:00:00.000Z").to("2021-05-12T21:00:00.000Z"))
+            .setPreFilterShardSize(5)
+            .get();
+
+        // THEN
+        assertEquals(RestStatus.OK, searchResponse.status());
+        assertEquals(idleIndexShardsCount + activeIndexShardsCount - 1, searchResponse.getSkippedShards());
+        assertEquals(0, searchResponse.getFailedShards());
+        Arrays.stream(searchResponse.getHits().getHits()).forEach(searchHit -> assertEquals("test2", searchHit.getIndex()));
+        // NOTE: we need an empty result from at least one shard
+        assertEquals(1, searchResponse.getHits().getHits().length);
+        final IndicesStatsResponse idleIndexStatsAfter = client().admin().indices().prepareStats(idleIndex).get();
+        assertIdleShardsRefreshStats(idleIndexStatsBefore, idleIndexStatsAfter);
+    }
+
+    public void testSearchIdleExistsQueryMatchOneIndex() throws InterruptedException {
+        // GIVEN
+        final String idleIndex = "test1";
+        final String activeIndex = "test2";
+        // NOTE: we need many shards because shard pre-filtering and the "can match" phase
+        // are executed only if we have enough shards.
+        int idleIndexShardsCount = 3;
+        int activeIndexShardsCount = 3;
+        createIndex(
+            idleIndex,
+            Settings.builder()
+                .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, idleIndexShardsCount)
+                .build(),
+            "doc",
+            "keyword",
+            "type=keyword"
+        );
+        createIndex(
+            activeIndex,
+            Settings.builder()
+                .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, activeIndexShardsCount)
+                .build(),
+            "doc",
+            "keyword",
+            "type=keyword"
+        );
+
+        assertEquals(RestStatus.CREATED, client().prepareIndex(idleIndex).setSource("keyword", "idle").get().status());
+        assertEquals(
+            RestStatus.CREATED,
+            client().prepareIndex(activeIndex).setSource("keyword", "active", "unmapped", "bbb").get().status()
+        );
+        assertEquals(RestStatus.OK, client().admin().indices().prepareRefresh(idleIndex, activeIndex).get().getStatus());
+
+        waitUntil(
+            () -> Arrays.stream(client().admin().indices().prepareStats(idleIndex, activeIndex).get().getShards())
+                .allMatch(ShardStats::isSearchIdle),
+            2,
+            TimeUnit.SECONDS
+        );
+
+        final IndicesStatsResponse idleIndexStatsBefore = client().admin().indices().prepareStats("test1").get();
+        assertIdleShard(idleIndexStatsBefore);
+
+        final IndicesStatsResponse activeIndexStatsBefore = client().admin().indices().prepareStats("test2").get();
+        assertIdleShard(activeIndexStatsBefore);
+
+        // WHEN
+        final SearchResponse searchResponse = client().prepareSearch("test*")
+            .setQuery(new ExistsQueryBuilder("unmapped"))
+            .setPreFilterShardSize(5)
+            .get();
+
+        // THEN
+        assertEquals(RestStatus.OK, searchResponse.status());
+        assertEquals(idleIndexShardsCount, searchResponse.getSkippedShards());
+        assertEquals(0, searchResponse.getFailedShards());
+        Arrays.stream(searchResponse.getHits().getHits()).forEach(searchHit -> assertEquals("test2", searchHit.getIndex()));
+        // NOTE: we need an empty result from at least one shard
+        assertEquals(1, searchResponse.getHits().getHits().length);
+        final IndicesStatsResponse idleIndexStatsAfter = client().admin().indices().prepareStats(idleIndex).get();
+        assertIdleShardsRefreshStats(idleIndexStatsBefore, idleIndexStatsAfter);
+    }
+
+    private static void assertIdleShard(final IndicesStatsResponse statsResponse) {
+        Arrays.stream(statsResponse.getShards()).forEach(shardStats -> assertTrue(shardStats.isSearchIdle()));
+        Arrays.stream(statsResponse.getShards()).forEach(shardStats -> assertTrue(shardStats.getSearchIdleTime() >= 100));
+    }
+
+    private static void assertIdleShardsRefreshStats(final IndicesStatsResponse before, final IndicesStatsResponse after) {
+        assertNotEquals(0, before.getShards().length);
+        assertNotEquals(0, after.getShards().length);
+        final List<RefreshStats> refreshStatsBefore = Arrays.stream(before.getShards()).map(x -> x.getStats().refresh).toList();
+        final List<RefreshStats> refreshStatsAfter = Arrays.stream(after.getShards()).map(x -> x.getStats().refresh).toList();
+        assertEquals(refreshStatsBefore.size(), refreshStatsAfter.size());
+        assertTrue(refreshStatsAfter.containsAll(refreshStatsBefore));
+        assertTrue(refreshStatsBefore.containsAll(refreshStatsAfter));
+    }
 }

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -49,9 +50,14 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsAccounting;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.MappingParserContext;
 import org.elasticsearch.index.mapper.NodeMappingStats;
+import org.elasticsearch.index.mapper.RuntimeField;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.query.SearchIndexNameMatcher;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
@@ -76,6 +82,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
 import java.io.Closeable;
@@ -661,6 +668,46 @@ public SearchExecutionContext newSearchExecutionContext(
         );
     }
 
+    /**
+     * Creates a new {@link QueryRewriteContext}.
+     * This class is used to rewrite queries in case access to the index is not required, since we can
+     * decide rewriting based on mappings alone. This saves the cost of pulling an index searcher as
+     * well as the associated cost of refreshing idle shards.
+     */
+    public QueryRewriteContext newQueryRewriteContext(
+        final LongSupplier nowInMillis,
+        final Map<String, Object> runtimeMappings,
+        final String clusterAlias
+    ) {
+        final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(
+            index().getName(),
+            clusterAlias,
+            clusterService,
+            expressionResolver
+        );
+        final MapperService mapperService = mapperService();
+        final MappingLookup mappingLookup = mapperService().mappingLookup();
+        return new QueryRewriteContext(
+            parserConfiguration,
+            client,
+            nowInMillis,
+            mapperService,
+            mappingLookup,
+            parseRuntimeMappings(runtimeMappings, mapperService, indexSettings, mappingLookup),
+            null,
+            indexSettings,
+            new Index(
+                RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
+                indexSettings.getIndex().getUUID()
+            ),
+            indexNameMatcher,
+            namedWriteableRegistry,
+            valuesSourceRegistry,
+            allowExpensiveQueries,
+            scriptService
+        );
+    }
+
     /**
      * The {@link ThreadPool} to use for this index.
      */
@@ -1212,4 +1259,28 @@ public boolean clearCaches(boolean queryCache, boolean fieldDataCache, String...
         return clearedAtLeastOne;
     }
 
+    public static Map<String, MappedFieldType> parseRuntimeMappings(
+        Map<String, Object> runtimeMappings,
+        MapperService mapperService,
+        IndexSettings indexSettings,
+        MappingLookup lookup
+    ) {
+        if (runtimeMappings.isEmpty()) {
+            return Collections.emptyMap();
+        }
+        // TODO add specific tests to SearchExecutionTests similar to the ones in FieldTypeLookupTests
+        MappingParserContext parserContext = mapperService.parserContext();
+        Map<String, RuntimeField> runtimeFields = RuntimeField.parseRuntimeFields(new HashMap<>(runtimeMappings), parserContext, false);
+        Map<String, MappedFieldType> runtimeFieldTypes = RuntimeField.collectFieldTypes(runtimeFields.values());
+        if (false == indexSettings.getIndexMetadata().getRoutingPaths().isEmpty()) {
+            for (String r : runtimeMappings.keySet()) {
+                if (Regex.simpleMatch(indexSettings.getIndexMetadata().getRoutingPaths(), r)) {
+                    throw new IllegalArgumentException("runtime fields may not match [routing_path] but [" + r + "] matched");
+                }
+            }
+        }
+        runtimeFieldTypes.keySet().forEach(lookup::validateDoesNotShadow);
+        return runtimeFieldTypes;
+    }
+
 }

server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java

@@ -45,7 +45,6 @@
 import org.elasticsearch.index.mapper.MappingParserContext;
 import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.RuntimeField;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.query.support.NestedScope;
@@ -76,6 +75,8 @@
 import java.util.function.LongSupplier;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.index.IndexService.parseRuntimeMappings;
+
 /**
  * The context used to execute a search request on a shard. It provides access
  * to required information like mapping definitions and document data.
@@ -615,13 +616,15 @@ public IndexSettings getIndexSettings() {
     }
 
     /** Return the current {@link IndexReader}, or {@code null} if no index reader is available,
-     *  for instance if this rewrite context is used to index queries (percolation). */
+     *  for instance if this rewrite context is used to index queries (percolation).
+     */
     public IndexReader getIndexReader() {
         return searcher == null ? null : searcher.getIndexReader();
     }
 
-    /** Return the current {@link IndexSearcher}, or {@code null} if no index reader is available,
-     *  for instance if this rewrite context is used to index queries (percolation). */
+    /** Return the current {@link IndexSearcher}, or {@code null} if no index reader is available, which happens
+     * if this rewrite context is used to index queries (percolation).
+     */
     public IndexSearcher searcher() {
         return searcher;
     }
@@ -645,30 +648,6 @@ public boolean fieldExistsInIndex(String fieldname) {
         return fieldsInIndex.contains(fieldname);
     }
 
-    private static Map<String, MappedFieldType> parseRuntimeMappings(
-        Map<String, Object> runtimeMappings,
-        MapperService mapperService,
-        IndexSettings indexSettings,
-        MappingLookup lookup
-    ) {
-        if (runtimeMappings.isEmpty()) {
-            return Collections.emptyMap();
-        }
-        // TODO add specific tests to SearchExecutionTests similar to the ones in FieldTypeLookupTests
-        MappingParserContext parserContext = mapperService.parserContext();
-        Map<String, RuntimeField> runtimeFields = RuntimeField.parseRuntimeFields(new HashMap<>(runtimeMappings), parserContext, false);
-        Map<String, MappedFieldType> runtimeFieldTypes = RuntimeField.collectFieldTypes(runtimeFields.values());
-        if (false == indexSettings.getIndexMetadata().getRoutingPaths().isEmpty()) {
-            for (String r : runtimeMappings.keySet()) {
-                if (Regex.simpleMatch(indexSettings.getIndexMetadata().getRoutingPaths(), r)) {
-                    throw new IllegalArgumentException("runtime fields may not match [routing_path] but [" + r + "] matched");
-                }
-            }
-        }
-        runtimeFieldTypes.keySet().forEach(lookup::validateDoesNotShadow);
-        return runtimeFieldTypes;
-    }
-
     /**
      * Cache key for current mapping.
      */