[7.x] Deprecate Auto-Follow system indices (#73237)

This commits deprecates Auto-Follow of system indices

Relates #72815

docs/reference/migration/migrate_7_14.asciidoc

@@ -132,4 +132,23 @@ Configuring a realm name with a leading underscore is deprecated. In a future re
 it will result in an error on startup if any user configured realm has a name
 with a leading underscore.
 ====
+
+[discrete]
+[[breaking_714_ccr_changes]]
+==== CCR deprecations
+
+[[system-indices-auto-follow-deprecation]]
+.Auto-follow remote system indices is deprecated.
+[%collapsible]
+====
+*Details* +
+Currently, remote system indices matching an <<ccr-auto-follow,auto-follow pattern>>
+are configured as a follower index automatically, this behavior is deprecated.
+
+*Impact* +
+In 8.0.0, remote system indices matching an <<ccr-auto-follow,auto-follow pattern>>
+won't be configured as a follower index automatically. In order to adapt to this new
+behaviour it is advised to exclude patterns matching system indices such as `.tasks` and
+`kibana-*`.
+====
 // end::notable-breaking-changes[]

x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java

@@ -11,31 +11,42 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.xpack.CcrIntegTestCase;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
+import org.elasticsearch.xpack.core.ccr.CcrAutoFollowInfoFetcher;
+import org.elasticsearch.xpack.core.ccr.CcrConstants;
 import org.elasticsearch.xpack.core.ccr.action.ActivateAutoFollowPatternAction;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo;
 import org.elasticsearch.xpack.core.ccr.action.FollowParameters;
 import org.elasticsearch.xpack.core.ccr.action.GetAutoFollowPatternAction;
+import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction;
 import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -46,6 +57,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -60,6 +72,49 @@ protected boolean reuseClusters() {
         return false;
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return org.elasticsearch.core.List.of(FakeSystemIndexPlugin.class, SecondFakeSystemIndexPlugin.class);
+    }
+
+    public static class FakeSystemIndexPlugin extends Plugin implements SystemIndexPlugin {
+        public static final String SYSTEM_INDEX_NAME = ".test-system-idx";
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "test"));
+        }
+
+        @Override
+        public String getFeatureName() {
+            return "FakeSystemIndexPlugin";
+        }
+
+        @Override
+        public String getFeatureDescription() {
+            return "FakeSystemIndexPlugin";
+        }
+    }
+
+    public static class SecondFakeSystemIndexPlugin extends Plugin implements SystemIndexPlugin {
+        public static final String SYSTEM_INDEX_NAME = ".another-test-system-idx";
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "test"));
+        }
+
+        @Override
+        public String getFeatureName() {
+            return "SecondFakeSystemIndexPlugin";
+        }
+
+        @Override
+        public String getFeatureDescription() {
+            return "Fake system index";
+        }
+    }
+
     public void testAutoFollow() throws Exception {
         Settings leaderIndexSettings = Settings.builder()
             .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
@@ -117,8 +172,8 @@ public void testCleanFollowedLeaderIndexUUIDs() throws Exception {
 
             Metadata metadata = getFollowerCluster().clusterService().state().metadata();
             String leaderIndexUUID = metadata.index("copy-logs-201901")
-                .getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY)
-                .get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY);
+                .getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY)
+                .get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY);
             AutoFollowMetadata autoFollowMetadata = metadata.custom(AutoFollowMetadata.TYPE);
             assertThat(autoFollowMetadata, notNullValue());
             List<String> followedLeaderIndixUUIDs = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("my-pattern");
@@ -573,7 +628,6 @@ public void testAutoFollowExclusion() throws Exception {
             .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
             .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
             .build();
-
         putAutoFollowPatterns("my-pattern1", new String[] {"logs-*"}, Collections.singletonList("logs-2018*"));
 
         createLeaderIndex("logs-201801", leaderIndexSettings);
@@ -594,6 +648,69 @@ public void testAutoFollowExclusion() throws Exception {
         assertFalse(indexExists("copy-logs-201801", followerClient()));
     }
 
+    public void testGetAutoFollowedSystemIndices() throws Exception {
+        assertThat(getFollowerAutoFollowedSystemIndices(), is(empty()));
+
+        // This index is created before the auto-follow pattern therefore it won't be auto-followed
+        // but it's in the followedLeaderIndexUUIDs list anyway.
+        createLeaderSystemIndex(FakeSystemIndexPlugin.SYSTEM_INDEX_NAME);
+
+        putAutoFollowPatterns("my-pattern", new String[]{".*", "logs-*"});
+
+        assertLongBusy(() -> {
+            final AutoFollowStats autoFollowStats = getAutoFollowStats();
+            assertThat(autoFollowStats.getAutoFollowedClusters().size(), equalTo(1));
+            assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L));
+        });
+
+        assertThat(getFollowerAutoFollowedSystemIndices(), is(empty()));
+
+        Settings leaderIndexSettings = Settings.builder()
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            .build();
+        createLeaderIndex("logs-202101", leaderIndexSettings);
+        createLeaderSystemIndex(SecondFakeSystemIndexPlugin.SYSTEM_INDEX_NAME);
+
+        final String followerSystemIndexName = "copy-" + SecondFakeSystemIndexPlugin.SYSTEM_INDEX_NAME;
+
+        ensureFollowerGreen(followerSystemIndexName);
+        ensureFollowerGreen("copy-logs-202101");
+
+        assertLongBusy(() -> {
+            final AutoFollowStats autoFollowStats = getAutoFollowStats();
+            assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(2L));
+
+            // Ensure that the operations have been replicated
+            final GetResponse response = followerClient().prepareGet(followerSystemIndexName, "_doc", "1").execute().actionGet();
+            assertThat(response.isExists(), equalTo(true));
+        });
+
+        final List<String> autoFollowedIndices = getFollowerAutoFollowedSystemIndices();
+        assertThat(autoFollowedIndices.size(), is(equalTo(1)));
+        assertThat(autoFollowedIndices.get(0), is(equalTo(followerSystemIndexName)));
+
+        followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(followerSystemIndexName)).actionGet();
+
+        assertLongBusy(() -> {
+            assertThat(getFollowerAutoFollowedSystemIndices(), is(empty()));
+        });
+    }
+
+    private void createLeaderSystemIndex(String indexName) {
+        leaderClient().index(new IndexRequest(indexName).id("1").source("completed", true)).actionGet();
+        final GetResponse getResponse = leaderClient().prepareGet(indexName, "_doc", "1").execute().actionGet();
+        assertThat(getResponse.isExists(), equalTo(true));
+    }
+
+    private List<String> getFollowerAutoFollowedSystemIndices() {
+        final ClusterService followerClusterService = getFollowerCluster().getMasterNodeInstance(ClusterService.class);
+        PlainActionFuture<List<String>> future = PlainActionFuture.newFuture();
+        CcrAutoFollowInfoFetcher.getAutoFollowedSystemIndices(followerClient(), followerClusterService.state(), future);
+        return future.actionGet();
+    }
+
     private boolean indexExists(String index, Client client) {
         return client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet().isExists();
     }

x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java

@@ -59,6 +59,7 @@
 import org.elasticsearch.xpack.ccr.action.repositories.PutCcrRestoreSessionAction;
 import org.elasticsearch.xpack.ccr.repository.CcrRepository;
 import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService;
+import org.elasticsearch.xpack.core.ccr.CcrConstants;
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
 
 import java.io.IOException;
@@ -185,10 +186,10 @@ public void testThatRepositoryRecoversEmptyIndexBasedOnLeaderSettings() throws I
         IndexMetadata leaderMetadata = leaderState.getState().metadata().index(leaderIndex);
         IndexMetadata followerMetadata = followerState.getState().metadata().index(followerIndex);
         assertEquals(leaderMetadata.getNumberOfShards(), followerMetadata.getNumberOfShards());
-        Map<String, String> ccrMetadata = followerMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY);
-        assertEquals(leaderIndex, ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY));
-        assertEquals(leaderMetadata.getIndexUUID(), ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY));
-        assertEquals("leader_cluster", ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY));
+        Map<String, String> ccrMetadata = followerMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY);
+        assertEquals(leaderIndex, ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY));
+        assertEquals(leaderMetadata.getIndexUUID(), ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY));
+        assertEquals("leader_cluster", ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY));
         assertEquals(followerIndex, followerMetadata.getSettings().get(IndexMetadata.SETTING_INDEX_PROVIDED_NAME));
 
         // UUID is changed so that we can follow indexes on same cluster

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java

@@ -132,11 +132,7 @@
 public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, EnginePlugin, RepositoryPlugin, ClusterPlugin {
 
     public static final String CCR_THREAD_POOL_NAME = "ccr";
-    public static final String CCR_CUSTOM_METADATA_KEY = "ccr";
-    public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS = "leader_index_shard_history_uuids";
-    public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY = "leader_index_uuid";
-    public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY = "leader_index_name";
-    public static final String CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY = "remote_cluster_name";
+    // Constants have been moved into CcrConstants
 
     public static final String REQUESTED_OPS_MISSING_METADATA_KEY = "es.requested_operations_missing";
 

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java

@@ -30,6 +30,8 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -38,12 +40,12 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.transport.NoSuchRemoteClusterException;
-import org.elasticsearch.xpack.ccr.Ccr;
 import org.elasticsearch.xpack.ccr.CcrLicenseChecker;
 import org.elasticsearch.xpack.ccr.CcrSettings;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
+import org.elasticsearch.xpack.core.ccr.CcrConstants;
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
 import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants;
 
@@ -75,6 +77,8 @@
 public class AutoFollowCoordinator extends AbstractLifecycleComponent implements ClusterStateListener {
 
     private static final Logger LOGGER = LogManager.getLogger(AutoFollowCoordinator.class);
+    public static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(AutoFollowCoordinator.class);
+
     private static final int MAX_AUTO_FOLLOW_ERRORS = 256;
 
     private final Client client;
@@ -538,6 +542,14 @@ private void checkAutoFollowPattern(String autoFollowPattenName,
                         updateAutoFollowMetadata(recordLeaderIndexAsFollowFunction(autoFollowPattenName, indexToFollow),
                             error -> groupedListener.onResponse(new Tuple<>(indexToFollow, error)));
                     } else {
+                        if (indexAbstraction.isSystem()) {
+                            deprecationLogger.deprecate(DeprecationCategory.INDICES,
+                                "ccr_auto_follow_system_indices",
+                                "Auto following a leader system index " + indexToFollow.getName() +
+                                    " will not work in the next major version"
+                            );
+                        }
+
                         followLeaderIndex(autoFollowPattenName, remoteCluster, indexToFollow, autoFollowPattern, headers,
                             error -> groupedListener.onResponse(new Tuple<>(indexToFollow, error)));
                     }
@@ -555,9 +567,9 @@ private static boolean leaderIndexAlreadyFollowed(AutoFollowPattern autoFollowPa
                 // we should let the auto follower attempt to auto follow it, so it can fail later and
                 // it is then visible in the auto follow stats. For example a cluster can just happen to have
                 // an index with the same name as the new follower index.
-                Map<String, String> customData = indexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY);
+                Map<String, String> customData = indexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY);
                 if (customData != null) {
-                    String recordedLeaderIndexUUID = customData.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY);
+                    String recordedLeaderIndexUUID = customData.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY);
                     return leaderIndex.getUUID().equals(recordedLeaderIndexUUID);
                 }
             }

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java

@@ -71,6 +71,7 @@
 import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsAction;
 import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsRequest;
 import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsResponse;
+import org.elasticsearch.xpack.core.ccr.CcrConstants;
 import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask;
 
 import java.util.ArrayList;
@@ -521,7 +522,7 @@ private void logRetentionLeaseFailure(final String retentionLeaseId, final Throw
 
     private String getLeaderShardHistoryUUID(ShardFollowTask params) {
         IndexMetadata followIndexMetadata = clusterService.state().metadata().index(params.getFollowShardId().getIndex());
-        Map<String, String> ccrIndexMetadata = followIndexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY);
+        Map<String, String> ccrIndexMetadata = followIndexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY);
         String[] recordedLeaderShardHistoryUUIDs = extractLeaderShardHistoryUUIDs(ccrIndexMetadata);
         return recordedLeaderShardHistoryUUIDs[params.getLeaderShardId().id()];
     }

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java

@@ -20,7 +20,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.ccr.Ccr;
+import org.elasticsearch.xpack.core.ccr.CcrConstants;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.Status;
@@ -65,7 +65,7 @@ static List<FollowerInfo> getFollowInfos(List<String> concreteFollowerIndices, C
 
         for (String index : concreteFollowerIndices) {
             IndexMetadata indexMetadata = state.metadata().index(index);
-            Map<String, String> ccrCustomData = indexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY);
+            Map<String, String> ccrCustomData = indexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY);
             if (ccrCustomData != null) {
                 Optional<ShardFollowTask> result;
                 if (persistentTasks != null) {
@@ -78,8 +78,8 @@ static List<FollowerInfo> getFollowInfos(List<String> concreteFollowerIndices, C
                 }
 
                 String followerIndex = indexMetadata.getIndex().getName();
-                String remoteCluster = ccrCustomData.get(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY);
-                String leaderIndex = ccrCustomData.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY);
+                String remoteCluster = ccrCustomData.get(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY);
+                String leaderIndex = ccrCustomData.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY);
                 if (result.isPresent()) {
                     ShardFollowTask params = result.get();
                     FollowParameters followParameters = new FollowParameters();