Auto convert shared_cache SBIs to only use frozen tier (elastic#71014)

This commit converts the index metadata of searchable snapshot indices using the `shared_cache`
storage type to:

- Remove all the `index.routing.allocation.(include|exclude|require)._tier` settings
- Sets `index.routing.allocation.include._tier_preference` to `data_frozen` automatically when the index metadata is read

This is in preperation to enforcing that the `_tier_preference` setting is always set to
`data_frozen` for shared cache SBIs.

Relates to elastic#70846, elastic#71013, elastic#70786, elastic#70141

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

@@ -71,9 +71,11 @@ public IndexMetadataVerifier(Settings settings, NamedXContentRegistry xContentRe
     public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, Version minimumIndexCompatibilityVersion) {
         checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion);
 
-        // we have to run this first otherwise in we try to create IndexSettings
-        // with broken settings and fail in checkMappingsCompatibility
-        IndexMetadata newMetadata = archiveBrokenIndexSettings(indexMetadata);
+        // First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen
+        IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata);
+        // Next we have to run this otherwise if we try to create IndexSettings
+        // with broken settings it would fail in checkMappingsCompatibility
+        newMetadata = archiveBrokenIndexSettings(newMetadata);
         checkMappingsCompatibility(newMetadata);
         return newMetadata;
     }
@@ -183,4 +185,30 @@ IndexMetadata archiveBrokenIndexSettings(IndexMetadata indexMetadata) {
             return indexMetadata;
         }
     }
+
+    /**
+     * Convert shared_cache searchable snapshot indices to only specify
+     * _tier_preference: data_frozen, removing any pre-existing tier allocation rules.
+     */
+    IndexMetadata convertSharedCacheTierPreference(IndexMetadata indexMetadata) {
+        final Settings settings = indexMetadata.getSettings();
+        // Only remove these settings for a shared_cache searchable snapshot
+        if ("snapshot".equals(settings.get("index.store.type", "")) && settings.getAsBoolean("index.store.snapshot.partial", false)) {
+            final Settings.Builder settingsBuilder = Settings.builder().put(settings);
+            // Clear any allocation rules other than preference for tier
+            settingsBuilder.remove("index.routing.allocation.include._tier");
+            settingsBuilder.remove("index.routing.allocation.exclude._tier");
+            settingsBuilder.remove("index.routing.allocation.require._tier");
+            // Override the tier preference to be only on frozen nodes, regardless of its current setting
+            settingsBuilder.put("index.routing.allocation.include._tier_preference", "data_frozen");
+            final Settings newSettings = settingsBuilder.build();
+            if (settings.equals(newSettings)) {
+                return indexMetadata;
+            } else {
+                return IndexMetadata.builder(indexMetadata).settings(newSettings).build();
+            }
+        } else {
+            return indexMetadata;
+        }
+    }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataConversionTests.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.mapper.MapperRegistry;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
+
+import java.util.Collections;
+
+import static org.elasticsearch.test.VersionUtils.randomIndexCompatibleVersion;
+import static org.hamcrest.Matchers.equalTo;
+
+public class IndexMetadataConversionTests extends ESTestCase {
+
+    public void testConvertSearchableSnapshotSettings() {
+        IndexMetadataVerifier service = getIndexMetadataVerifier();
+        IndexMetadata src = newIndexMeta("foo", Settings.EMPTY);
+        IndexMetadata indexMetadata = service.convertSharedCacheTierPreference(src);
+        assertSame(indexMetadata, src);
+
+        // A full_copy searchable snapshot (settings should be untouched)
+        src = newIndexMeta("foo", Settings.builder()
+            .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
+            .put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
+            .put("index.routing.allocation.include._tier", "data_hot")
+            .put("index.routing.allocation.exclude._tier", "data_warm")
+            .put("index.routing.allocation.require._tier", "data_hot")
+            .put("index.routing.allocation.include._tier_preference", "data_cold")
+            .build());
+        indexMetadata = service.convertSharedCacheTierPreference(src);
+        assertSame(indexMetadata, src);
+
+        // A shared_cache searchable snapshot with valid settings (metadata should be untouched)
+        src = newIndexMeta("foo", Settings.builder()
+            .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
+            .put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
+            .put("index.routing.allocation.include._tier_preference", "data_frozen")
+            .build());
+        indexMetadata = service.convertSharedCacheTierPreference(src);
+        assertSame(indexMetadata, src);
+
+        // A shared_cache searchable snapshot (should have its settings converted)
+        src = newIndexMeta("foo", Settings.builder()
+            .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
+            .put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), true)
+            .put("index.routing.allocation.include._tier", "data_hot")
+            .put("index.routing.allocation.exclude._tier", "data_warm")
+            .put("index.routing.allocation.require._tier", "data_hot")
+            .put("index.routing.allocation.include._tier_preference", "data_frozen,data_cold")
+            .build());
+        indexMetadata = service.convertSharedCacheTierPreference(src);
+        assertNotSame(indexMetadata, src);
+        Settings newSettings = indexMetadata.getSettings();
+        assertNull(newSettings.get("index.routing.allocation.include._tier"));
+        assertNull(newSettings.get("index.routing.allocation.exclude._tier"));
+        assertNull(newSettings.get("index.routing.allocation.require._tier"));
+        assertThat(newSettings.get("index.routing.allocation.include._tier_preference"), equalTo("data_frozen"));
+    }
+
+    private IndexMetadataVerifier getIndexMetadataVerifier() {
+        return new IndexMetadataVerifier(
+            Settings.EMPTY,
+            xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(),
+                MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            null
+        );
+    }
+
+    public static IndexMetadata newIndexMeta(String name, Settings indexSettings) {
+        final Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, randomIndexCompatibleVersion(random()))
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5))
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))
+            .put(IndexMetadata.SETTING_CREATION_DATE, randomNonNegativeLong())
+            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
+            .put(indexSettings)
+            .build();
+        final IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(name).settings(settings);
+        if (randomBoolean()) {
+            indexMetadataBuilder.state(IndexMetadata.State.CLOSE);
+        }
+        return indexMetadataBuilder.build();
+    }
+}

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java

@@ -502,12 +502,7 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
             expectedReplicas = 0;
         }
         final String expectedDataTiersPreference;
-        if (randomBoolean()) {
-            expectedDataTiersPreference = String.join(",", randomSubsetOf(DataTier.ALL_DATA_TIERS));
-            indexSettingsBuilder.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, expectedDataTiersPreference);
-        } else {
-            expectedDataTiersPreference = getDataTiersPreference(MountSearchableSnapshotRequest.Storage.SHARED_CACHE);
-        }
+        expectedDataTiersPreference = getDataTiersPreference(MountSearchableSnapshotRequest.Storage.SHARED_CACHE);
 
         indexSettingsBuilder.put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.ZERO);
         final AtomicBoolean statsWatcherRunning = new AtomicBoolean(true);