From bd02f91a58f38097ac49b9a67410de716733abe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Tue, 29 Jun 2021 15:18:01 +0200 Subject: [PATCH] [7.x] Deprecate Auto-Follow system indices (#73237) This commits deprecates Auto-Follow of system indices Relates #72815 --- .../reference/migration/migrate_7_14.asciidoc | 19 +++ .../elasticsearch/xpack/ccr/AutoFollowIT.java | 125 +++++++++++++++- .../xpack/ccr/CcrRepositoryIT.java | 9 +- .../java/org/elasticsearch/xpack/ccr/Ccr.java | 6 +- .../ccr/action/AutoFollowCoordinator.java | 18 ++- .../ccr/action/ShardFollowTasksExecutor.java | 3 +- .../ccr/action/TransportFollowInfoAction.java | 8 +- .../action/TransportPauseFollowAction.java | 4 +- .../action/TransportResumeFollowAction.java | 14 +- .../ccr/action/TransportUnfollowAction.java | 14 +- .../xpack/ccr/repository/CcrRepository.java | 19 ++- .../xpack/ccr/CCRFeatureSetTests.java | 3 +- .../action/AutoFollowCoordinatorTests.java | 116 +++++++++++++- .../TransportFollowInfoActionTests.java | 4 +- .../TransportResumeFollowActionTests.java | 14 +- .../action/TransportUnfollowActionTests.java | 12 +- .../core/ccr/CcrAutoFollowInfoFetcher.java | 141 ++++++++++++++++++ .../xpack/core/ccr/CcrConstants.java | 18 +++ .../CcrAutoFollowedSystemIndicesChecker.java | 53 +++++++ .../xpack/deprecation/DeprecationChecker.java | 10 +- .../TransportDeprecationInfoAction.java | 6 +- .../TransportDeprecationInfoActionTests.java | 4 +- 22 files changed, 550 insertions(+), 70 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrAutoFollowInfoFetcher.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrConstants.java create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/CcrAutoFollowedSystemIndicesChecker.java diff --git a/docs/reference/migration/migrate_7_14.asciidoc b/docs/reference/migration/migrate_7_14.asciidoc index 9808abe74ac98..a03530aae4c88 100644 --- a/docs/reference/migration/migrate_7_14.asciidoc +++ b/docs/reference/migration/migrate_7_14.asciidoc @@ -132,4 +132,23 @@ Configuring a realm name with a leading underscore is deprecated. In a future re it will result in an error on startup if any user configured realm has a name with a leading underscore. ==== + +[discrete] +[[breaking_714_ccr_changes]] +==== CCR deprecations + +[[system-indices-auto-follow-deprecation]] +.Auto-follow remote system indices is deprecated. +[%collapsible] +==== +*Details* + +Currently, remote system indices matching an <> +are configured as a follower index automatically, this behavior is deprecated. + +*Impact* + +In 8.0.0, remote system indices matching an <> +won't be configured as a follower index automatically. In order to adapt to this new +behaviour it is advised to exclude patterns matching system indices such as `.tasks` and +`kibana-*`. +==== // end::notable-breaking-changes[] diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index de46ad0f578d4..49bc3ff644f2e 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -11,21 +11,30 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.core.CheckedRunnable; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.xpack.CcrIntegTestCase; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowStats; +import org.elasticsearch.xpack.core.ccr.CcrAutoFollowInfoFetcher; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.ActivateAutoFollowPatternAction; import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction; @@ -33,9 +42,11 @@ import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo; import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import org.elasticsearch.xpack.core.ccr.action.GetAutoFollowPatternAction; +import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -46,6 +57,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; @@ -60,6 +72,49 @@ protected boolean reuseClusters() { return false; } + @Override + protected Collection> nodePlugins() { + return org.elasticsearch.core.List.of(FakeSystemIndexPlugin.class, SecondFakeSystemIndexPlugin.class); + } + + public static class FakeSystemIndexPlugin extends Plugin implements SystemIndexPlugin { + public static final String SYSTEM_INDEX_NAME = ".test-system-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "test")); + } + + @Override + public String getFeatureName() { + return "FakeSystemIndexPlugin"; + } + + @Override + public String getFeatureDescription() { + return "FakeSystemIndexPlugin"; + } + } + + public static class SecondFakeSystemIndexPlugin extends Plugin implements SystemIndexPlugin { + public static final String SYSTEM_INDEX_NAME = ".another-test-system-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "test")); + } + + @Override + public String getFeatureName() { + return "SecondFakeSystemIndexPlugin"; + } + + @Override + public String getFeatureDescription() { + return "Fake system index"; + } + } + public void testAutoFollow() throws Exception { Settings leaderIndexSettings = Settings.builder() .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) @@ -117,8 +172,8 @@ public void testCleanFollowedLeaderIndexUUIDs() throws Exception { Metadata metadata = getFollowerCluster().clusterService().state().metadata(); String leaderIndexUUID = metadata.index("copy-logs-201901") - .getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY) - .get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); + .getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY) + .get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); AutoFollowMetadata autoFollowMetadata = metadata.custom(AutoFollowMetadata.TYPE); assertThat(autoFollowMetadata, notNullValue()); List followedLeaderIndixUUIDs = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("my-pattern"); @@ -573,7 +628,6 @@ public void testAutoFollowExclusion() throws Exception { .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) .build(); - putAutoFollowPatterns("my-pattern1", new String[] {"logs-*"}, Collections.singletonList("logs-2018*")); createLeaderIndex("logs-201801", leaderIndexSettings); @@ -594,6 +648,69 @@ public void testAutoFollowExclusion() throws Exception { assertFalse(indexExists("copy-logs-201801", followerClient())); } + public void testGetAutoFollowedSystemIndices() throws Exception { + assertThat(getFollowerAutoFollowedSystemIndices(), is(empty())); + + // This index is created before the auto-follow pattern therefore it won't be auto-followed + // but it's in the followedLeaderIndexUUIDs list anyway. + createLeaderSystemIndex(FakeSystemIndexPlugin.SYSTEM_INDEX_NAME); + + putAutoFollowPatterns("my-pattern", new String[]{".*", "logs-*"}); + + assertLongBusy(() -> { + final AutoFollowStats autoFollowStats = getAutoFollowStats(); + assertThat(autoFollowStats.getAutoFollowedClusters().size(), equalTo(1)); + assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L)); + }); + + assertThat(getFollowerAutoFollowedSystemIndices(), is(empty())); + + Settings leaderIndexSettings = Settings.builder() + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) + .build(); + createLeaderIndex("logs-202101", leaderIndexSettings); + createLeaderSystemIndex(SecondFakeSystemIndexPlugin.SYSTEM_INDEX_NAME); + + final String followerSystemIndexName = "copy-" + SecondFakeSystemIndexPlugin.SYSTEM_INDEX_NAME; + + ensureFollowerGreen(followerSystemIndexName); + ensureFollowerGreen("copy-logs-202101"); + + assertLongBusy(() -> { + final AutoFollowStats autoFollowStats = getAutoFollowStats(); + assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(2L)); + + // Ensure that the operations have been replicated + final GetResponse response = followerClient().prepareGet(followerSystemIndexName, "_doc", "1").execute().actionGet(); + assertThat(response.isExists(), equalTo(true)); + }); + + final List autoFollowedIndices = getFollowerAutoFollowedSystemIndices(); + assertThat(autoFollowedIndices.size(), is(equalTo(1))); + assertThat(autoFollowedIndices.get(0), is(equalTo(followerSystemIndexName))); + + followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(followerSystemIndexName)).actionGet(); + + assertLongBusy(() -> { + assertThat(getFollowerAutoFollowedSystemIndices(), is(empty())); + }); + } + + private void createLeaderSystemIndex(String indexName) { + leaderClient().index(new IndexRequest(indexName).id("1").source("completed", true)).actionGet(); + final GetResponse getResponse = leaderClient().prepareGet(indexName, "_doc", "1").execute().actionGet(); + assertThat(getResponse.isExists(), equalTo(true)); + } + + private List getFollowerAutoFollowedSystemIndices() { + final ClusterService followerClusterService = getFollowerCluster().getMasterNodeInstance(ClusterService.class); + PlainActionFuture> future = PlainActionFuture.newFuture(); + CcrAutoFollowInfoFetcher.getAutoFollowedSystemIndices(followerClient(), followerClusterService.state(), future); + return future.actionGet(); + } + private boolean indexExists(String index, Client client) { return client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet().isExists(); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 55ea46afc8f31..60c4771b0d5d6 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -59,6 +59,7 @@ import org.elasticsearch.xpack.ccr.action.repositories.PutCcrRestoreSessionAction; import org.elasticsearch.xpack.ccr.repository.CcrRepository; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import java.io.IOException; @@ -185,10 +186,10 @@ public void testThatRepositoryRecoversEmptyIndexBasedOnLeaderSettings() throws I IndexMetadata leaderMetadata = leaderState.getState().metadata().index(leaderIndex); IndexMetadata followerMetadata = followerState.getState().metadata().index(followerIndex); assertEquals(leaderMetadata.getNumberOfShards(), followerMetadata.getNumberOfShards()); - Map ccrMetadata = followerMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); - assertEquals(leaderIndex, ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY)); - assertEquals(leaderMetadata.getIndexUUID(), ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY)); - assertEquals("leader_cluster", ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY)); + Map ccrMetadata = followerMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); + assertEquals(leaderIndex, ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY)); + assertEquals(leaderMetadata.getIndexUUID(), ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY)); + assertEquals("leader_cluster", ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY)); assertEquals(followerIndex, followerMetadata.getSettings().get(IndexMetadata.SETTING_INDEX_PROVIDED_NAME)); // UUID is changed so that we can follow indexes on same cluster diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index 6538dd80ea18d..c69c54e69b37a 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -132,11 +132,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, EnginePlugin, RepositoryPlugin, ClusterPlugin { public static final String CCR_THREAD_POOL_NAME = "ccr"; - public static final String CCR_CUSTOM_METADATA_KEY = "ccr"; - public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS = "leader_index_shard_history_uuids"; - public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY = "leader_index_uuid"; - public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY = "leader_index_name"; - public static final String CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY = "remote_cluster_name"; + // Constants have been moved into CcrConstants public static final String REQUESTED_OPS_MISSING_METADATA_KEY = "es.requested_operations_missing"; diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java index 9b935b78a18cb..12599c1fae70a 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java @@ -30,6 +30,8 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; @@ -38,12 +40,12 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.transport.NoSuchRemoteClusterException; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; import org.elasticsearch.xpack.core.ccr.AutoFollowStats; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants; @@ -75,6 +77,8 @@ public class AutoFollowCoordinator extends AbstractLifecycleComponent implements ClusterStateListener { private static final Logger LOGGER = LogManager.getLogger(AutoFollowCoordinator.class); + public static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(AutoFollowCoordinator.class); + private static final int MAX_AUTO_FOLLOW_ERRORS = 256; private final Client client; @@ -538,6 +542,14 @@ private void checkAutoFollowPattern(String autoFollowPattenName, updateAutoFollowMetadata(recordLeaderIndexAsFollowFunction(autoFollowPattenName, indexToFollow), error -> groupedListener.onResponse(new Tuple<>(indexToFollow, error))); } else { + if (indexAbstraction.isSystem()) { + deprecationLogger.deprecate(DeprecationCategory.INDICES, + "ccr_auto_follow_system_indices", + "Auto following a leader system index " + indexToFollow.getName() + + " will not work in the next major version" + ); + } + followLeaderIndex(autoFollowPattenName, remoteCluster, indexToFollow, autoFollowPattern, headers, error -> groupedListener.onResponse(new Tuple<>(indexToFollow, error))); } @@ -555,9 +567,9 @@ private static boolean leaderIndexAlreadyFollowed(AutoFollowPattern autoFollowPa // we should let the auto follower attempt to auto follow it, so it can fail later and // it is then visible in the auto follow stats. For example a cluster can just happen to have // an index with the same name as the new follower index. - Map customData = indexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); + Map customData = indexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); if (customData != null) { - String recordedLeaderIndexUUID = customData.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); + String recordedLeaderIndexUUID = customData.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); return leaderIndex.getUUID().equals(recordedLeaderIndexUUID); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index 6f38e232265ee..79449faed308d 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -71,6 +71,7 @@ import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsAction; import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsRequest; import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsResponse; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; import java.util.ArrayList; @@ -521,7 +522,7 @@ private void logRetentionLeaseFailure(final String retentionLeaseId, final Throw private String getLeaderShardHistoryUUID(ShardFollowTask params) { IndexMetadata followIndexMetadata = clusterService.state().metadata().index(params.getFollowShardId().getIndex()); - Map ccrIndexMetadata = followIndexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); + Map ccrIndexMetadata = followIndexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); String[] recordedLeaderShardHistoryUUIDs = extractLeaderShardHistoryUUIDs(ccrIndexMetadata); return recordedLeaderShardHistoryUUIDs[params.getLeaderShardId().id()]; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java index 9cac18a560f10..89efb409d09a2 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java @@ -20,7 +20,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.ccr.Ccr; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.Status; @@ -65,7 +65,7 @@ static List getFollowInfos(List concreteFollowerIndices, C for (String index : concreteFollowerIndices) { IndexMetadata indexMetadata = state.metadata().index(index); - Map ccrCustomData = indexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); + Map ccrCustomData = indexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); if (ccrCustomData != null) { Optional result; if (persistentTasks != null) { @@ -78,8 +78,8 @@ static List getFollowInfos(List concreteFollowerIndices, C } String followerIndex = indexMetadata.getIndex().getName(); - String remoteCluster = ccrCustomData.get(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY); - String leaderIndex = ccrCustomData.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); + String remoteCluster = ccrCustomData.get(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY); + String leaderIndex = ccrCustomData.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); if (result.isPresent()) { ShardFollowTask params = result.get(); FollowParameters followParameters = new FollowParameters(); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java index 877f7945bfa29..3112788962f9e 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.ccr.Ccr; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction; import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; @@ -56,7 +56,7 @@ protected void masterOperation(PauseFollowAction.Request request, listener.onFailure(new IndexNotFoundException(request.getFollowIndex())); return; } - if (followerIMD.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY) == null) { + if (followerIMD.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY) == null) { listener.onFailure(new IllegalArgumentException("index [" + request.getFollowIndex() + "] is not a follower index")); return; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index 94b8d535359c0..da276bb2dce54 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -44,10 +44,10 @@ import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; @@ -121,14 +121,14 @@ protected void masterOperation(final ResumeFollowAction.Request request, return; } - final Map ccrMetadata = followerIndexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); + final Map ccrMetadata = followerIndexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); if (ccrMetadata == null) { throw new IllegalArgumentException("follow index ["+ request.getFollowerIndex() + "] does not have ccr metadata"); } - final String leaderCluster = ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY); + final String leaderCluster = ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY); // Validates whether the leader cluster has been configured properly: client.getRemoteClusterClient(leaderCluster); - final String leaderIndex = ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); + final String leaderIndex = ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); ccrLicenseChecker.checkRemoteClusterLicenseAndFetchLeaderIndexMetadataAndHistoryUUIDs( client, leaderCluster, @@ -183,12 +183,12 @@ static void validate( final MapperService followerMapperService) { FollowParameters parameters = request.getParameters(); - Map ccrIndexMetadata = followIndex.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); + Map ccrIndexMetadata = followIndex.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); if (ccrIndexMetadata == null) { throw new IllegalArgumentException("follow index ["+ followIndex.getIndex().getName() + "] does not have ccr metadata"); } String leaderIndexUUID = leaderIndex.getIndex().getUUID(); - String recordedLeaderIndexUUID = ccrIndexMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); + String recordedLeaderIndexUUID = ccrIndexMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); if (leaderIndexUUID.equals(recordedLeaderIndexUUID) == false) { throw new IllegalArgumentException("follow index [" + request.getFollowerIndex() + "] should reference [" + leaderIndexUUID + "] as leader index but instead reference [" + recordedLeaderIndexUUID + "] as leader index"); @@ -352,7 +352,7 @@ private static ShardFollowTask createShardFollowTask( } static String[] extractLeaderShardHistoryUUIDs(Map ccrIndexMetadata) { - String historyUUIDs = ccrIndexMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS); + String historyUUIDs = ccrIndexMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS); if (historyUUIDs == null) { throw new IllegalArgumentException("leader index shard UUIDs are missing"); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java index e9d7d5f38d823..b53f7482d19fb 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java @@ -38,9 +38,9 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrRetentionLeases; import org.elasticsearch.xpack.ccr.CcrSettings; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; import org.elasticsearch.xpack.core.ccr.action.UnfollowAction; @@ -95,11 +95,11 @@ public void onFailure(final String source, final Exception e) { @Override public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { final IndexMetadata indexMetadata = oldState.metadata().index(request.getFollowerIndex()); - final Map ccrCustomMetadata = indexMetadata.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); - final String remoteClusterName = ccrCustomMetadata.get(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY); + final Map ccrCustomMetadata = indexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); + final String remoteClusterName = ccrCustomMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY); - final String leaderIndexName = ccrCustomMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); - final String leaderIndexUuid = ccrCustomMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); + final String leaderIndexName = ccrCustomMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); + final String leaderIndexUuid = ccrCustomMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); final Index leaderIndex = new Index(leaderIndexName, leaderIndexUuid); final String retentionLeaseId = CcrRetentionLeases.retentionLeaseId( oldState.getClusterName().value(), @@ -222,7 +222,7 @@ static ClusterState unfollow(String followerIndex, ClusterState current) { throw new IndexNotFoundException(followerIndex); } - if (followerIMD.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY) == null) { + if (followerIMD.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY) == null) { throw new IllegalArgumentException("index [" + followerIndex + "] is not a follower index"); } @@ -253,7 +253,7 @@ static ClusterState unfollow(String followerIndex, ClusterState current) { newIndexMetadata.settings(builder); newIndexMetadata.settingsVersion(followerIMD.getSettingsVersion() + 1); // Remove ccr custom metadata - newIndexMetadata.removeCustom(Ccr.CCR_CUSTOM_METADATA_KEY); + newIndexMetadata.removeCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY); Metadata newMetadata = Metadata.builder(current.metadata()) .put(newIndexMetadata) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 5fa281acb1f2b..e0b9ada0c8102 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -90,6 +90,7 @@ import org.elasticsearch.xpack.ccr.action.repositories.GetCcrRestoreFileChunkRequest; import org.elasticsearch.xpack.ccr.action.repositories.PutCcrRestoreSessionAction; import org.elasticsearch.xpack.ccr.action.repositories.PutCcrRestoreSessionRequest; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import java.io.Closeable; import java.io.IOException; @@ -230,11 +231,11 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna IndexMetadata.Builder imdBuilder = IndexMetadata.builder(leaderIndex); // Adding the leader index uuid for each shard as custom metadata: Map metadata = new HashMap<>(); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, String.join(",", leaderHistoryUUIDs)); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, leaderIndexMetadata.getIndexUUID()); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY, leaderIndexMetadata.getIndex().getName()); - metadata.put(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY, remoteClusterAlias); - imdBuilder.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, metadata); + metadata.put(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, String.join(",", leaderHistoryUUIDs)); + metadata.put(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, leaderIndexMetadata.getIndexUUID()); + metadata.put(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY, leaderIndexMetadata.getIndex().getName()); + metadata.put(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY, remoteClusterAlias); + imdBuilder.putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, metadata); imdBuilder.settings(leaderIndexMetadata.getSettings()); @@ -355,9 +356,11 @@ public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, Sh // TODO: Add timeouts to network calls / the restore process. createEmptyStore(store); - final Map ccrMetadata = store.indexSettings().getIndexMetadata().getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); - final String leaderIndexName = ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); - final String leaderUUID = ccrMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); + final Map ccrMetadata = store.indexSettings() + .getIndexMetadata() + .getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); + final String leaderIndexName = ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); + final String leaderUUID = ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); final Index leaderIndex = new Index(leaderIndexName, leaderUUID); final ShardId leaderShardId = new ShardId(leaderIndex, shardId.getId()); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java index 95d16958c51a0..6dd5403a45ede 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.junit.Before; import org.mockito.Mockito; @@ -86,7 +87,7 @@ public void testUsageStats() throws Exception { .numberOfShards(1) .numberOfReplicas(0) .creationDate(i) - .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + .putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); metadata.put(followerIndex); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index 12b4dc1dbad78..fa8b26cb0749c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -13,8 +13,8 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -34,13 +34,13 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.ccr.action.AutoFollowCoordinator.AutoFollower; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; import org.elasticsearch.xpack.core.ccr.AutoFollowStats; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.ActivateAutoFollowPatternAction; import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; @@ -1813,8 +1813,9 @@ public void testAutoFollowerFollowerIndexAlreadyExists() { .metadata(Metadata.builder() .put(IndexMetadata.builder("logs-20190101") .settings(settings(Version.CURRENT)) - .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, Collections.singletonMap(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, - remoteState.metadata().index("logs-20190101").getIndexUUID())) + .putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, + Collections.singletonMap(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, + remoteState.metadata().index("logs-20190101").getIndexUUID())) .numberOfShards(1) .numberOfReplicas(0)) .putCustom(AutoFollowMetadata.TYPE, autoFollowMetadata)) @@ -2154,6 +2155,113 @@ void cleanFollowedRemoteIndices(ClusterState remoteClusterState, List pa } } + public void testDeprecationWarningsAreEmittedWhenASystemIndexIsAutoFollowed() throws Exception { + final Client client = mock(Client.class); + when(client.getRemoteClusterClient(anyString())).thenReturn(client); + + final String pattern = "pattern1"; + final ClusterState localState = ClusterState.builder(new ClusterName("local")) + .metadata(Metadata.builder() + .putCustom(AutoFollowMetadata.TYPE, + new AutoFollowMetadata( + Collections.singletonMap( + pattern, + new AutoFollowPattern( + "remote", + Collections.singletonList(".*"), + Collections.emptyList(), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ), + Collections.singletonMap(pattern, Collections.emptyList()), + Collections.singletonMap(pattern, Collections.emptyMap())))) + .build(); + + final Metadata.Builder metadataBuilder = Metadata.builder(); + final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + final int nbLeaderSystemIndices = randomIntBetween(1, 10); + for (int i = 0; i < nbLeaderSystemIndices; i++) { + final String indexName = ".system-" + i; + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings(settings(Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))) + .numberOfShards(1) + .numberOfReplicas(0) + .system(true) + .build(); + metadataBuilder.put(indexMetadata, true); + + routingTableBuilder.add(IndexRoutingTable.builder(indexMetadata.getIndex()) + .addShard(TestShardRouting.newShardRouting(indexName, 0, "1", true, ShardRoutingState.INITIALIZING).moveToStarted()) + .build()); + } + + final ClusterState remoteState = ClusterState.builder(new ClusterName("remote")) + .metadata(metadataBuilder.build()) + .routingTable(routingTableBuilder.build()) + .build(); + + final List results = new ArrayList<>(); + final Set followedIndices = ConcurrentCollections.newConcurrentSet(); + final AutoFollower autoFollower = + new AutoFollower("remote", results::addAll, localClusterStateSupplier(localState), () -> 1L, Runnable::run) { + @Override + void getRemoteClusterState(String remoteCluster, + long metadataVersion, + BiConsumer handler) { + assertThat(remoteCluster, equalTo("remote")); + handler.accept(new ClusterStateResponse(new ClusterName("remote"), remoteState, false), null); + } + + @Override + void createAndFollow(Map headers, + PutFollowAction.Request followRequest, + Runnable successHandler, + Consumer failureHandler) { + followedIndices.add(followRequest.getLeaderIndex()); + successHandler.run(); + } + + @Override + void updateAutoFollowMetadata(Function updateFunction, Consumer handler) { + handler.accept(null); + } + + @Override + void cleanFollowedRemoteIndices(ClusterState remoteClusterState, List patterns) { + // Ignore, to avoid invoking updateAutoFollowMetadata(...) twice + } + }; + autoFollower.start(); + + assertThat(results, notNullValue()); + assertThat(results.size(), equalTo(1)); + + List expectedDeprecationWarnings = new ArrayList<>(nbLeaderSystemIndices); + for (ObjectObjectCursor index : remoteState.metadata().indices()) { + assertThat(results.get(0).autoFollowExecutionResults.containsKey(index.value.getIndex()), is(true)); + assertThat(followedIndices.contains(index.key), is(true)); + final String indexName = index.value.getIndex().getName(); + expectedDeprecationWarnings.add( + "Auto following a leader system index " + indexName + " will not work in the next major version" + ); + } + + assertWarnings(expectedDeprecationWarnings.toArray(new String[0])); + } + private static ClusterState createRemoteClusterState(String indexName, Boolean enableSoftDeletes) { return createRemoteClusterState(indexName, enableSoftDeletes, 0L); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoActionTests.java index ea21cb3f19df1..e31172e630a8e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoActionTests.java @@ -15,7 +15,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ccr.Ccr; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo; import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; @@ -59,7 +59,7 @@ private static ClusterState createCS(String[] indices, boolean[] followerIndices .numberOfReplicas(0); if (isFollowIndex) { - imdBuilder.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + imdBuilder.putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); if (active) { persistentTasks.addTask(Integer.toString(i), ShardFollowTask.NAME, createShardFollowTask(new Index(index, IndexMetadata.INDEX_UUID_NA_VALUE)), null); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 545d11a166585..37569230df01d 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -20,8 +20,8 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import java.io.IOException; @@ -47,8 +47,8 @@ public static ResumeFollowAction.Request resumeFollow(String followerIndex) { public void testValidation() throws IOException { final Map customMetadata = new HashMap<>(); - customMetadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, "uuid"); - customMetadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, "_na_"); + customMetadata.put(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, "uuid"); + customMetadata.put(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, "_na_"); ResumeFollowAction.Request request = resumeFollow("index2"); String[] UUIDs = new String[]{"uuid"}; @@ -63,7 +63,7 @@ public void testValidation() throws IOException { // should fail because the recorded leader index uuid is not equal to the leader actual index IndexMetadata leaderIMD = createIMD("index1", 5, Settings.EMPTY, null); IndexMetadata followIMD = createIMD("index2", 5, Settings.EMPTY, - singletonMap(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, "another-value")); + singletonMap(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, "another-value")); Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null)); assertThat(e.getMessage(), equalTo("follow index [index2] should reference [_na_] as leader index but " + @@ -73,8 +73,8 @@ public void testValidation() throws IOException { // should fail because the recorded leader index history uuid is not equal to the leader actual index history uuid: IndexMetadata leaderIMD = createIMD("index1", 5, Settings.EMPTY, null); Map anotherCustomMetadata = new HashMap<>(); - anotherCustomMetadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, "_na_"); - anotherCustomMetadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, "another-uuid"); + anotherCustomMetadata.put(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, "_na_"); + anotherCustomMetadata.put(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, "another-uuid"); IndexMetadata followIMD = createIMD("index2", 5, Settings.EMPTY, anotherCustomMetadata); Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null)); @@ -280,7 +280,7 @@ private static IndexMetadata createIMD(String index, .putMapping("_doc", mapping); if (custom != null) { - builder.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, custom); + builder.putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, custom); } return builder.build(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java index cd641ea201667..2c4a7908efc03 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; +import org.elasticsearch.xpack.core.ccr.CcrConstants; import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; import java.util.Collections; @@ -39,7 +39,7 @@ public void testUnfollow() { .numberOfShards(1) .numberOfReplicas(0) .state(IndexMetadata.State.CLOSE) - .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + .putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); ClusterState current = ClusterState.builder(new ClusterName("cluster_name")) .metadata(Metadata.builder() @@ -50,7 +50,7 @@ public void testUnfollow() { IndexMetadata resultIMD = result.metadata().index("follow_index"); assertThat(resultIMD.getSettings().get(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey()), nullValue()); - assertThat(resultIMD.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY), nullValue()); + assertThat(resultIMD.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY), nullValue()); assertThat(resultIMD.getSettingsVersion(), equalTo(settingsVersion + 1)); } @@ -59,7 +59,7 @@ public void testUnfollowIndexOpen() { .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)) .numberOfShards(1) .numberOfReplicas(0) - .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + .putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); ClusterState current = ClusterState.builder(new ClusterName("cluster_name")) .metadata(Metadata.builder() @@ -77,7 +77,7 @@ public void testUnfollowRunningShardFollowTasks() { .numberOfShards(1) .numberOfReplicas(0) .state(IndexMetadata.State.CLOSE) - .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + .putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); ShardFollowTask params = new ShardFollowTask( @@ -116,7 +116,7 @@ public void testUnfollowMissingIndex() { .numberOfShards(1) .numberOfReplicas(0) .state(IndexMetadata.State.CLOSE) - .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + .putCustom(CcrConstants.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); ClusterState current = ClusterState.builder(new ClusterName("cluster_name")) .metadata(Metadata.builder() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrAutoFollowInfoFetcher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrAutoFollowInfoFetcher.java new file mode 100644 index 0000000000000..1d30b821babdc --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrAutoFollowInfoFetcher.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ccr; + +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.index.Index; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class CcrAutoFollowInfoFetcher { + private CcrAutoFollowInfoFetcher() {} + + public static void getAutoFollowedSystemIndices(Client client, ClusterState state, ActionListener> listener) { + final AutoFollowMetadata autoFollowMetadata = state.metadata().custom(AutoFollowMetadata.TYPE); + final PersistentTasksCustomMetadata persistentTasks = state.metadata().custom(PersistentTasksCustomMetadata.TYPE); + + if (autoFollowMetadata == null || areShardFollowTasksRunning(persistentTasks) == false) { + listener.onResponse(Collections.emptyList()); + return; + } + + final List followedLeaderIndexUUIDs = autoFollowMetadata.getFollowedLeaderIndexUUIDs() + .values() + .stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + final Map> remoteClusterAutoFollowedIndices = new HashMap<>(); + for (ObjectObjectCursor indexEntry : state.metadata().getIndices()) { + final IndexMetadata indexMetadata = indexEntry.value; + final Map ccrMetadata = indexMetadata.getCustomData(CcrConstants.CCR_CUSTOM_METADATA_KEY); + if (ccrMetadata == null) { + continue; + } + + final String leaderIndexUUID = ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); + final String leaderIndexName = ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); + final String remoteCluster = ccrMetadata.get(CcrConstants.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY); + + final Index followerIndex = indexMetadata.getIndex(); + if (followedLeaderIndexUUIDs.contains(leaderIndexUUID) && isCurrentlyFollowed(persistentTasks, followerIndex)) { + List autoFollowedIndices = + remoteClusterAutoFollowedIndices.computeIfAbsent(remoteCluster, unused -> new ArrayList<>()); + + autoFollowedIndices.add(new AutoFollowedIndex(leaderIndexName, followerIndex.getName())); + } + } + + if (remoteClusterAutoFollowedIndices.isEmpty()) { + listener.onResponse(Collections.emptyList()); + return; + } + + GroupedActionListener> clusterResponsesListener = new GroupedActionListener<>( + listener.map((clusterAutoFollowedSystemIndices) -> + clusterAutoFollowedSystemIndices.stream() + .flatMap(Collection::stream) + .collect(Collectors.toList())), + remoteClusterAutoFollowedIndices.size() + ); + + // When a system index is followed we don't copy over the isSystem flag into IndexMetadata + // We need to fetch the remote cluster state in order to check whether or not the following index + // follows a leader system index. + for (Map.Entry> remoteAutoFollowedIndicesEntry : remoteClusterAutoFollowedIndices.entrySet()) { + final String remoteCluster = remoteAutoFollowedIndicesEntry.getKey(); + final List autoFollowedIndices = remoteAutoFollowedIndicesEntry.getValue(); + + try { + client.getRemoteClusterClient(remoteCluster) + .admin() + .cluster() + .prepareState() + .clear() + .setMetadata(true) + .execute(new ActionListener() { + @Override + public void onResponse(ClusterStateResponse stateResponse) { + final ClusterState clusterState = stateResponse.getState(); + List autoFollowedSystemIndices = new ArrayList<>(); + for (AutoFollowedIndex autoFollowedIndex : autoFollowedIndices) { + final IndexAbstraction indexAbstraction = + clusterState.metadata().getIndicesLookup().get(autoFollowedIndex.remoteIndexName); + if (indexAbstraction != null && indexAbstraction.isSystem()) { + autoFollowedSystemIndices.add(autoFollowedIndex.localIndexName); + } + } + clusterResponsesListener.onResponse(autoFollowedSystemIndices); + } + + @Override + public void onFailure(Exception e) { + clusterResponsesListener.onFailure(e); + } + }); + } catch (IllegalArgumentException e) { + clusterResponsesListener.onFailure(e); + } + } + } + + private static boolean isCurrentlyFollowed(PersistentTasksCustomMetadata persistentTasks, Index index) { + return persistentTasks != null && persistentTasks.findTasks(ShardFollowTask.NAME, task -> true).stream() + .map(task -> (ShardFollowTask) task.getParams()) + .anyMatch(shardFollowTask -> index.equals(shardFollowTask.getFollowShardId().getIndex())); + } + + private static boolean areShardFollowTasksRunning(PersistentTasksCustomMetadata persistentTasks) { + return persistentTasks != null && persistentTasks.findTasks(ShardFollowTask.NAME, task -> true).isEmpty() == false; + } + + private static class AutoFollowedIndex { + private final String remoteIndexName; + private final String localIndexName; + + private AutoFollowedIndex(String remoteIndexName, String localIndexName) { + this.remoteIndexName = remoteIndexName; + this.localIndexName = localIndexName; + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrConstants.java new file mode 100644 index 0000000000000..46373465fb0a1 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CcrConstants.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ccr; + +public class CcrConstants { + public static final String CCR_CUSTOM_METADATA_KEY = "ccr"; + public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY = "leader_index_uuid"; + public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS = "leader_index_shard_history_uuids"; + public static final String CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY = "leader_index_name"; + public static final String CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY = "remote_cluster_name"; + + private CcrConstants() {} +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/CcrAutoFollowedSystemIndicesChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/CcrAutoFollowedSystemIndicesChecker.java new file mode 100644 index 0000000000000..20c5f75b403a1 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/CcrAutoFollowedSystemIndicesChecker.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.ccr.CcrAutoFollowInfoFetcher; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.XPackSettings.CCR_ENABLED_SETTING; + +public class CcrAutoFollowedSystemIndicesChecker implements DeprecationChecker { + @Override + public boolean enabled(Settings settings) { + return CCR_ENABLED_SETTING.get(settings); + } + + @Override + public void check(Components components, ActionListener deprecationIssueListener) { + CcrAutoFollowInfoFetcher.getAutoFollowedSystemIndices( + components.client(), + components.clusterState(), + deprecationIssueListener.map(autoFollowedSystemIndices -> { + final List deprecationIssues = autoFollowedSystemIndices.stream() + .map(this::createDeprecationIssue) + .collect(Collectors.toList()); + return new CheckResult(getName(), deprecationIssues); + }) + ); + } + + private DeprecationIssue createDeprecationIssue(String localIndexName) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "An auto followed index follows a remote system index", + "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/migrating-7.14.html#breaking_714_ccr_changes", + "Auto followed index [" + localIndexName + + "] follows a remote system index and this behaviour will change in the next major version.", + null + ); + } + + @Override + public String getName() { + return "ccr_auto_followed_system_indices"; + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecker.java index c0cbe68995d71..854d949477925 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecker.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -61,13 +62,16 @@ class Components { private final NamedXContentRegistry xContentRegistry; private final Settings settings; private final Client client; + private final ClusterState clusterState; - Components(NamedXContentRegistry xContentRegistry, Settings settings, OriginSettingClient client) { + Components(NamedXContentRegistry xContentRegistry, Settings settings, OriginSettingClient client, ClusterState clusterState) { this.xContentRegistry = xContentRegistry; this.settings = settings; this.client = client; + this.clusterState = clusterState; } + public NamedXContentRegistry xContentRegistry() { return xContentRegistry; } @@ -80,5 +84,9 @@ public Client client() { return client; } + public ClusterState clusterState() { + return clusterState; + } + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 574c455dbc775..f3bfee88c7479 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -39,7 +39,8 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction { - private static final List PLUGIN_CHECKERS = Arrays.asList(new MlDeprecationChecker()); + private static final List PLUGIN_CHECKERS = + Arrays.asList(new MlDeprecationChecker(), new CcrAutoFollowedSystemIndicesChecker()); private static final Logger logger = LogManager.getLogger(TransportDeprecationInfoAction.class); private final NodeClient client; @@ -86,7 +87,8 @@ protected final void masterOperation(final DeprecationInfoAction.Request request DeprecationChecker.Components components = new DeprecationChecker.Components( xContentRegistry, settings, - new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN) + new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN), + state ); pluginSettingIssues(PLUGIN_CHECKERS, components, ActionListener.wrap( deprecationIssues -> { diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java index 9d931f0c2138f..55fc1bec41898 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoActionTests.java @@ -27,7 +27,7 @@ public class TransportDeprecationInfoActionTests extends ESTestCase { public void testPluginSettingIssues() { - DeprecationChecker.Components components = new DeprecationChecker.Components(null, Settings.EMPTY, null); + DeprecationChecker.Components components = new DeprecationChecker.Components(null, Settings.EMPTY, null, null); PlainActionFuture>> future = new PlainActionFuture<>(); TransportDeprecationInfoAction.pluginSettingIssues(Arrays.asList( new NamedChecker("foo", Collections.emptyList(), false), @@ -45,7 +45,7 @@ public void testPluginSettingIssues() { } public void testPluginSettingIssuesWithFailures() { - DeprecationChecker.Components components = new DeprecationChecker.Components(null, Settings.EMPTY, null); + DeprecationChecker.Components components = new DeprecationChecker.Components(null, Settings.EMPTY, null, null); PlainActionFuture>> future = new PlainActionFuture<>(); TransportDeprecationInfoAction.pluginSettingIssues(Arrays.asList( new NamedChecker("foo", Collections.emptyList(), false),