Avoid regular indices in frozen tier (#70141)

The frozen tier will be dedicated for partially cached searchable
snapshots. This PR ensures that we do not allow allocating regular
indices (including fully cached searchable snapshots) to the frozen
tier.

build.gradle

@@ -189,9 +189,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
+boolean bwc_tests_enabled = false
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = ""
+String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/70141"
 /*
  * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a
  * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable

x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.cluster.routing.allocation;
 
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -24,6 +25,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -267,6 +269,35 @@ public void testTierFilteringIgnoredByFilterAllocationDecider() {
             .get();
     }
 
+    public void testIllegalOnFrozen() {
+        startDataNode();
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> createIndex(index, Settings.builder()
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 0)
+                .put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, DataTier.DATA_FROZEN)
+                .build()));
+        assertThat(e.getMessage(), equalTo("[data_frozen] tier can only be used for partial searchable snapshots"));
+
+        String initialTier = randomFrom(DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_COLD);
+        createIndex(index, Settings.builder()
+            .put("index.number_of_shards", 1)
+            .put("index.number_of_replicas", 0)
+            .put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, initialTier)
+            .build());
+
+        IllegalArgumentException e2 = expectThrows(IllegalArgumentException.class, () -> updatePreference(DataTier.DATA_FROZEN));
+        assertThat(e2.getMessage(), equalTo("[data_frozen] tier can only be used for partial searchable snapshots"));
+
+        updatePreference(randomValueOtherThan(initialTier, () -> randomFrom(DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_COLD)));
+    }
+
+    private void updatePreference(String tier) {
+        client().admin().indices().updateSettings(new UpdateSettingsRequest(index)
+            .settings(Map.of(DataTierAllocationDecider.INDEX_ROUTING_PREFER, tier))).actionGet();
+    }
+
     private DataTiersFeatureSetUsage getUsage() {
         XPackUsageResponse usages = new XPackUsageRequestBuilder(client()).execute().actionGet();
         return usages.getUsages().stream()

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java

@@ -21,13 +21,21 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.xpack.core.DataTier;
+import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
 
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.DataTier.DATA_FROZEN;
+
 /**
  * The {@code DataTierAllocationDecider} is a custom allocation decider that behaves similar to the
  * {@link org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider}, however it
@@ -45,20 +53,21 @@ public class DataTierAllocationDecider extends AllocationDecider {
     public static final String INDEX_ROUTING_PREFER = "index.routing.allocation.include._tier_preference";
     public static final String INDEX_ROUTING_EXCLUDE = "index.routing.allocation.exclude._tier";
 
+    private static final DataTierValidator VALIDATOR = new DataTierValidator();
     public static final Setting<String> CLUSTER_ROUTING_REQUIRE_SETTING = Setting.simpleString(CLUSTER_ROUTING_REQUIRE,
         DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.NodeScope);
     public static final Setting<String> CLUSTER_ROUTING_INCLUDE_SETTING = Setting.simpleString(CLUSTER_ROUTING_INCLUDE,
         DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.NodeScope);
     public static final Setting<String> CLUSTER_ROUTING_EXCLUDE_SETTING = Setting.simpleString(CLUSTER_ROUTING_EXCLUDE,
         DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.NodeScope);
     public static final Setting<String> INDEX_ROUTING_REQUIRE_SETTING = Setting.simpleString(INDEX_ROUTING_REQUIRE,
-        DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
+        VALIDATOR, Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<String> INDEX_ROUTING_INCLUDE_SETTING = Setting.simpleString(INDEX_ROUTING_INCLUDE,
-        DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
+        VALIDATOR, Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<String> INDEX_ROUTING_EXCLUDE_SETTING = Setting.simpleString(INDEX_ROUTING_EXCLUDE,
-        DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
+        VALIDATOR, Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<String> INDEX_ROUTING_PREFER_SETTING = Setting.simpleString(INDEX_ROUTING_PREFER,
-        DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
+        VALIDATOR, Setting.Property.Dynamic, Setting.Property.IndexScope);
 
     private static void validateTierSetting(String setting) {
         if (Strings.hasText(setting)) {
@@ -71,6 +80,31 @@ private static void validateTierSetting(String setting) {
         }
     }
 
+    private static class DataTierValidator implements Setting.Validator<String> {
+        private static final Collection<Setting<?>> dependencies = List.of(IndexModule.INDEX_STORE_TYPE_SETTING,
+            SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING);
+
+        @Override
+        public void validate(String value) {
+            validateTierSetting(value);
+        }
+
+        @Override
+        public void validate(String value, Map<Setting<?>, Object> settings) {
+            if (Strings.hasText(value) && SearchableSnapshotsConstants.isPartialSearchableSnapshotIndex(settings) == false) {
+                String[] split = value.split(",");
+                if (Arrays.stream(split).anyMatch(DATA_FROZEN::equals)) {
+                    throw new IllegalArgumentException("[" + DATA_FROZEN + "] tier can only be used for partial searchable snapshots");
+                }
+            }
+        }
+
+        @Override
+        public Iterator<Setting<?>> settings() {
+            return dependencies.iterator();
+        }
+    }
+
     private volatile String clusterRequire;
     private volatile String clusterInclude;
     private volatile String clusterExclude;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstants.java

@@ -7,19 +7,41 @@
 package org.elasticsearch.xpack.searchablesnapshots;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.Map;
+
 import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 
 public class SearchableSnapshotsConstants {
     public static final String SNAPSHOT_DIRECTORY_FACTORY_KEY = "snapshot";
 
     public static final String SNAPSHOT_RECOVERY_STATE_FACTORY_KEY = "snapshot_prewarm";
+    public static final Setting<Boolean> SNAPSHOT_PARTIAL_SETTING = Setting.boolSetting(
+        "index.store.snapshot.partial",
+        false,
+        Setting.Property.IndexScope,
+        Setting.Property.PrivateIndex,
+        Setting.Property.NotCopyableOnResize
+    );
 
     public static boolean isSearchableSnapshotStore(Settings indexSettings) {
         return SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings));
     }
 
+    /**
+     * Based on a map from setting to value, do the settings represent a partial searchable snapshot index?
+     *
+     * Both index.store.type and index.store.snapshot.partial must be supplied.
+     */
+    public static boolean isPartialSearchableSnapshotIndex(Map<Setting<?>, Object> indexSettings) {
+        assert indexSettings.containsKey(INDEX_STORE_TYPE_SETTING) : "must include store type in map";
+        assert indexSettings.get(SNAPSHOT_PARTIAL_SETTING) != null : "partial setting must be non-null in map (has default value)";
+        return SNAPSHOT_DIRECTORY_FACTORY_KEY.equals(indexSettings.get(INDEX_STORE_TYPE_SETTING))
+            && (boolean) indexSettings.get(SNAPSHOT_PARTIAL_SETTING);
+    }
+
     public static final String CACHE_FETCH_ASYNC_THREAD_POOL_NAME = "searchable_snapshots_cache_fetch_async";
     public static final String CACHE_FETCH_ASYNC_THREAD_POOL_SETTING = "xpack.searchable_snapshots.cache_fetch_async_thread_pool";
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.cluster.routing.allocation;
 
+import joptsimple.internal.Strings;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
@@ -27,21 +28,26 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.EmptySnapshotsInfoService;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
 import org.elasticsearch.xpack.core.DataTier;
+import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.core.DataTier.DATA_FROZEN;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -704,6 +710,53 @@ public void testExistedClusterFilters() {
                 "tier filters [data_hot,data_warm]"));
     }
 
+    public void testFrozenIllegalForRegularIndices() {
+        List<String> tierList = new ArrayList<>(randomSubsetOf(DataTier.ALL_DATA_TIERS));
+        if (tierList.contains(DATA_FROZEN) == false) {
+            tierList.add(DATA_FROZEN);
+        }
+        Randomness.shuffle(tierList);
+
+        String value = Strings.join(tierList, ",");
+        Setting<String> setting = randomTierSetting();
+        Settings.Builder builder = Settings.builder().put(setting.getKey(), value);
+        if (randomBoolean()) {
+            builder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY);
+        }
+
+        Settings settings = builder.build();
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> setting.get(settings));
+        assertThat(exception.getMessage(), equalTo("[data_frozen] tier can only be used for partial searchable snapshots"));
+    }
+
+    public void testFrozenLegalForPartialSnapshot() {
+        List<String> tierList = new ArrayList<>(randomSubsetOf(DataTier.ALL_DATA_TIERS));
+        if (tierList.contains(DATA_FROZEN) == false) {
+            tierList.add(DATA_FROZEN);
+        }
+        Randomness.shuffle(tierList);
+
+        String value = Strings.join(tierList, ",");
+        Setting<String> setting = randomTierSetting();
+        Settings.Builder builder = Settings.builder().put(setting.getKey(), value);
+        builder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY);
+        builder.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), true);
+
+        Settings settings = builder.build();
+
+        // validate do not throw
+        assertThat(setting.get(settings), equalTo(value));
+    }
+
+    public Setting<String> randomTierSetting() {
+        //noinspection unchecked
+        return randomFrom(
+            DataTierAllocationDecider.INDEX_ROUTING_EXCLUDE_SETTING,
+            DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING,
+            DataTierAllocationDecider.INDEX_ROUTING_REQUIRE_SETTING,
+            DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING);
+    }
+
     private ClusterState prepareState(ClusterState initialState) {
         return prepareState(initialState, Settings.EMPTY);
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsConstantsTests.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.searchablesnapshots;
+
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class SearchableSnapshotsConstantsTests extends ESTestCase {
+
+    public void testIsPartialSearchableSnapshotIndex() {
+        assertThat(SearchableSnapshotsConstants.isPartialSearchableSnapshotIndex(
+            Map.of(IndexModule.INDEX_STORE_TYPE_SETTING, SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY,
+                SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING, false)),
+            is(false));
+
+        assertThat(SearchableSnapshotsConstants.isPartialSearchableSnapshotIndex(
+            Map.of(IndexModule.INDEX_STORE_TYPE_SETTING, "abc",
+                SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING, randomBoolean())),
+            is(false));
+
+        assertThat(SearchableSnapshotsConstants.isPartialSearchableSnapshotIndex(
+            Map.of(IndexModule.INDEX_STORE_TYPE_SETTING, SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY,
+                SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING, true)),
+            is(true));
+    }
+}

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java

@@ -143,7 +143,7 @@ protected void mountSnapshot(
             storage
         );
 
-        final RestoreSnapshotResponse restoreResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, mountRequest).get();
+        final RestoreSnapshotResponse restoreResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, mountRequest).actionGet();
         assertThat(restoreResponse.getRestoreInfo().successfulShards(), equalTo(getNumShards(restoredIndexName).numPrimaries));
         assertThat(restoreResponse.getRestoreInfo().failedShards(), equalTo(0));
     }