Warn when searching a frozen index. (elastic#78184)

The coordinating node should warn when a search request
is targeting a frozen index.

Relates to elastic#70192

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -33,6 +34,8 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -92,6 +95,10 @@
 
 public class TransportSearchAction extends HandledTransportAction<SearchRequest, SearchResponse> {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(TransportSearchAction.class);
+    public static final String FROZEN_INDICES_DEPRECATION_MESSAGE = "Searching frozen indices [{}] is deprecated." +
+        " Consider cold or frozen tiers in place of frozen indices. The frozen feature will be removed in a feature release.";
+
     /** The maximum number of shards for a single search request. */
     public static final Setting<Long> SHARD_COUNT_LIMIT_SETTING = Setting.longSetting(
             "action.search.shard_count.limit", Long.MAX_VALUE, 1L, Property.Dynamic, Property.NodeScope);
@@ -600,7 +607,23 @@ private Index[] resolveLocalIndices(OriginalIndices localIndices,
         if (localIndices == null) {
             return Index.EMPTY_ARRAY; //don't search on any local index (happens when only remote indices were specified)
         }
-        return indexNameExpressionResolver.concreteIndices(clusterState, localIndices, timeProvider.getAbsoluteStartMillis());
+
+        List<String> frozenIndices = null;
+        Index[] indices = indexNameExpressionResolver.concreteIndices(clusterState, localIndices, timeProvider.getAbsoluteStartMillis());
+        for (Index index : indices) {
+            IndexMetadata indexMetadata = clusterState.metadata().index(index);
+            if (indexMetadata.getSettings().getAsBoolean("index.frozen", false)) {
+                if (frozenIndices == null) {
+                    frozenIndices = new ArrayList<>();
+                }
+                frozenIndices.add(index.getName());
+            }
+        }
+        if (frozenIndices != null) {
+            DEPRECATION_LOGGER.critical(DeprecationCategory.INDICES, "search-frozen-indices", FROZEN_INDICES_DEPRECATION_MESSAGE,
+                String.join(",", frozenIndices));
+        }
+        return indices;
     }
 
     private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, SearchRequest searchRequest,

server/src/test/java/org/elasticsearch/search/SearchServiceTests.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
@@ -923,16 +924,18 @@ public void testExpandSearchThrottled() {
     }
 
     public void testExpandSearchFrozen() {
-        createIndex("frozen_index");
+        String indexName = "frozen_index";
+        createIndex(indexName);
         client().execute(
             InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.INSTANCE,
-            new InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.Request("frozen_index",
+            new InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.Request(indexName,
                 "index.frozen", "true"))
             .actionGet();
 
-        client().prepareIndex("frozen_index", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex(indexName, "_doc").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         assertHitCount(client().prepareSearch().get(), 0L);
         assertHitCount(client().prepareSearch().setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED).get(), 1L);
+        assertWarnings(TransportSearchAction.FROZEN_INDICES_DEPRECATION_MESSAGE.replace("{}", indexName));
     }
 
     public void testCreateReduceContext() {

x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -92,18 +93,18 @@ String openReaders(TimeValue keepAlive, String... indices) {
     }
 
     public void testCloseFreezeAndOpen() throws Exception {
-        createIndex("index", Settings.builder().put("index.number_of_shards", 2).build());
-        client().prepareIndex("index", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        client().prepareIndex("index", "_doc", "2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        client().prepareIndex("index", "_doc", "3").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new FreezeRequest("index")));
+        String indexName = "index";
+        createIndex(indexName, Settings.builder().put("index.number_of_shards", 2).build());
+        client().prepareIndex(indexName, "_doc").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex(indexName, "_doc").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex(indexName, "_doc").setId("3").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
         expectThrows(
             ClusterBlockException.class,
-            () -> client().prepareIndex("index", "_doc", "4").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get()
+            () -> client().prepareIndex(indexName, "_doc").setId("4").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get()
         );
         IndicesService indexServices = getInstanceFromNode(IndicesService.class);
-        Index index = resolveIndex("index");
+        Index index = resolveIndex(indexName);
         IndexService indexService = indexServices.indexServiceSafe(index);
         IndexShard shard = indexService.getShard(0);
         Engine engine = IndexShardTestCase.getEngine(shard);
@@ -142,7 +143,7 @@ public void testCloseFreezeAndOpen() throws Exception {
         } while (searchResponse.getHits().getHits().length > 0);
         client().prepareClearScroll().addScrollId(searchResponse.getScrollId()).get();
 
-        String pitId = openReaders(TimeValue.timeValueMinutes(1), "index");
+        String pitId = openReaders(TimeValue.timeValueMinutes(1), indexName);
         try {
             for (int from = 0; from < 3; from++) {
                 searchResponse = client().prepareSearch()
@@ -161,6 +162,7 @@ public void testCloseFreezeAndOpen() throws Exception {
                     assertFalse(((FrozenEngine) engine).isReaderOpen());
                 }
             }
+            assertWarnings(TransportSearchAction.FROZEN_INDICES_DEPRECATION_MESSAGE.replace("{}", indexName));
         } finally {
             client().execute(ClosePointInTimeAction.INSTANCE, new ClosePointInTimeRequest(pitId)).get();
         }
@@ -178,12 +180,13 @@ public void testSearchAndGetAPIsAreThrottled() throws Exception {
             .endObject()
             .endObject()
             .endObject();
-        createIndex("index", Settings.builder().put("index.number_of_shards", 2).build(), "_doc", mapping);
+        String indexName = "index";
+        createIndex(indexName, Settings.builder().put("index.number_of_shards", 2).build(), "_doc", mapping);
         for (int i = 0; i < 10; i++) {
-            client().prepareIndex("index", "_doc", "" + i).setSource("field", "foo bar baz").get();
+            client().prepareIndex(indexName, "_doc").setId("" + i).setSource("field", "foo bar baz").get();
         }
         XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new FreezeRequest("index")));
+        assertAcked(xPackClient.freeze(new FreezeRequest(indexName)));
         int numRequests = randomIntBetween(20, 50);
         int numRefreshes = 0;
         for (int i = 0; i < numRequests; i++) {
@@ -192,29 +195,30 @@ public void testSearchAndGetAPIsAreThrottled() throws Exception {
             // searcher and rewrite the request outside of the search-throttle thread pool
             switch (randomFrom(Arrays.asList(0, 1, 2))) {
                 case 0:
-                    client().prepareGet("index", "_doc", "" + randomIntBetween(0, 9)).get();
+                    client().prepareGet(indexName, "_doc", "" + randomIntBetween(0, 9)).get();
                     break;
                 case 1:
-                    client().prepareSearch("index")
+                    client().prepareSearch(indexName)
                         .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED)
                         .setSearchType(SearchType.QUERY_THEN_FETCH)
                         .get();
                     // in total 4 refreshes 1x query & 1x fetch per shard (we have 2)
                     numRefreshes += 3;
                     break;
                 case 2:
-                    client().prepareTermVectors("index", "_doc", "" + randomIntBetween(0, 9)).get();
+                    client().prepareTermVectors(indexName, "_doc", "" + randomIntBetween(0, 9)).get();
                     break;
                 case 3:
-                    client().prepareExplain("index", "_doc", "" + randomIntBetween(0, 9)).setQuery(new MatchAllQueryBuilder()).get();
+                    client().prepareExplain(indexName, "_doc", "" + randomIntBetween(0, 9)).setQuery(new MatchAllQueryBuilder()).get();
                     break;
 
                 default:
                     assert false;
             }
         }
-        IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setRefresh(true).get();
+        IndicesStatsResponse index = client().admin().indices().prepareStats(indexName).clear().setRefresh(true).get();
         assertEquals(numRefreshes, index.getTotal().refresh.getTotal());
+        assertWarnings(TransportSearchAction.FROZEN_INDICES_DEPRECATION_MESSAGE.replace("{}", indexName));
     }
 
     public void testFreezeAndUnfreeze() throws ExecutionException, InterruptedException {
@@ -296,28 +300,30 @@ public void testUnfreezeClosedIndices() throws ExecutionException, InterruptedEx
         assertHitCount(client().prepareSearch().get(), 1L);
     }
 
-    public void testFreezePattern() throws ExecutionException, InterruptedException {
-        createIndex("test-idx", Settings.builder().put("index.number_of_shards", 1).build());
-        client().prepareIndex("test-idx", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
+    public void testFreezePattern() throws Exception {
+        String indexName = "test-idx";
+        createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build());
+        client().prepareIndex(indexName, "_doc").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         createIndex("test-idx-1", Settings.builder().put("index.number_of_shards", 1).build());
         client().prepareIndex("test-idx-1", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new FreezeRequest("test-idx")));
-        assertIndexFrozen("test-idx");
+        assertAcked(xPackClient.freeze(new FreezeRequest(indexName)));
+        assertIndexFrozen(indexName);
 
-        IndicesStatsResponse index = client().admin().indices().prepareStats("test-idx").clear().setRefresh(true).get();
+        IndicesStatsResponse index = client().admin().indices().prepareStats(indexName).clear().setRefresh(true).get();
         assertEquals(0, index.getTotal().refresh.getTotal());
-        assertHitCount(client().prepareSearch("test-idx").setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED).get(), 1);
-        index = client().admin().indices().prepareStats("test-idx").clear().setRefresh(true).get();
+        assertHitCount(client().prepareSearch(indexName).setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED).get(), 1);
+        index = client().admin().indices().prepareStats(indexName).clear().setRefresh(true).get();
         assertEquals(1, index.getTotal().refresh.getTotal());
 
         assertAcked(xPackClient.freeze(new FreezeRequest("test*")));
-        assertIndexFrozen("test-idx");
+        assertIndexFrozen(indexName);
         assertIndexFrozen("test-idx-1");
-        index = client().admin().indices().prepareStats("test-idx").clear().setRefresh(true).get();
+        index = client().admin().indices().prepareStats(indexName).clear().setRefresh(true).get();
         assertEquals(1, index.getTotal().refresh.getTotal());
         index = client().admin().indices().prepareStats("test-idx-1").clear().setRefresh(true).get();
         assertEquals(0, index.getTotal().refresh.getTotal());
+        assertWarnings(TransportSearchAction.FROZEN_INDICES_DEPRECATION_MESSAGE.replace("{}", indexName));
     }
 
     public void testCanMatch() throws IOException, ExecutionException, InterruptedException {

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java

@@ -12,6 +12,7 @@
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
+import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -30,6 +31,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -472,19 +474,28 @@ protected static Number count(String index) throws IOException {
     protected static Map<String, Object> search(String index, QueryBuilder query, Boolean ignoreThrottled) throws IOException {
         final Request request = new Request(HttpPost.METHOD_NAME, '/' + index + "/_search");
         request.setJsonEntity(new SearchSourceBuilder().trackTotalHits(true).query(query).toString());
+
+        // If warning are returned than these must exist in this set:
+        Set<String> expectedWarnings = new HashSet<>();
+        expectedWarnings.add(TransportSearchAction.FROZEN_INDICES_DEPRECATION_MESSAGE.replace("{}", index));
         if (ignoreThrottled != null) {
             request.addParameter("ignore_throttled", ignoreThrottled.toString());
-            RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder()
-                .setWarningsHandler(
-                    warnings -> Collections.singletonList(
-                        "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. "
-                            + "Consider cold or frozen tiers in place of frozen indices."
-                    ).equals(warnings) == false
-                )
-                .build();
-            request.setOptions(requestOptions);
+            expectedWarnings.add(
+                "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. "
+                    + "Consider cold or frozen tiers in place of frozen indices."
+            );
         }
 
+        RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> {
+            for (String warning : warnings) {
+                if (expectedWarnings.contains(warning) == false) {
+                    return true;
+                }
+            }
+            return false;
+        }).build();
+        request.setOptions(requestOptions);
+
         final Response response = client().performRequest(request);
         assertThat(
             "Failed to execute search request on index [" + index + "]: " + response,

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/indices.freeze/10_basic.yml

@@ -25,6 +25,7 @@
 - do:
     warnings:
       - "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."
+      - "Searching frozen indices [test] is deprecated. Consider cold or frozen tiers in place of frozen indices. The frozen feature will be removed in a feature release."
     search:
       rest_total_hits_as_int: true
       index: test
@@ -70,6 +71,7 @@
 - do:
     warnings:
       - "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."
+      - "Searching frozen indices [test,test-01] is deprecated. Consider cold or frozen tiers in place of frozen indices. The frozen feature will be removed in a feature release."
     search:
       rest_total_hits_as_int: true
       index: _all
@@ -138,6 +140,7 @@
 - do:
     warnings:
       - "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."
+      - "Searching frozen indices [test] is deprecated. Consider cold or frozen tiers in place of frozen indices. The frozen feature will be removed in a feature release."
     search:
       rest_total_hits_as_int: true
       index: _all

x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java

@@ -993,7 +993,9 @@ public void testFrozenIndexAfterRestarted() throws Exception {
             assertNoFileBasedRecovery(index, n -> true);
             final Request request = new Request("GET", "/" + index + "/_search");
             request.setOptions(expectWarnings("[ignore_throttled] parameter is deprecated because frozen " +
-                "indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."));
+                "indices have been deprecated. Consider cold or frozen tiers in place of frozen indices.",
+                "Searching frozen indices [" + index + "] is deprecated. " +
+                    "Consider cold or frozen tiers in place of frozen indices. The frozen feature will be removed in a feature release."));
             request.addParameter("ignore_throttled", "false");
             assertThat(XContentMapValues.extractValue("hits.total.value", entityAsMap(client().performRequest(request))),
                 equalTo(totalHits));