From cf6b03c8f45c5f2c15d90e2435a6eaab8bf35153 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 11 Apr 2017 13:56:26 -0500 Subject: [PATCH] Wildcard cluster names for cross cluster search (#23985) This is related to #23893. This commit allows users to use wilcards for cluster names when executing a cross cluster search. So instead of defining every cluster such as: GET one:*,two:*,three:*/_search A user could just search: GET *:*/_search As ":" characters are currently allowed in index names, if the text up to the first ":" does not match a defined cluster name, the entire string is treated as an index name. --- .../action/search/RemoteClusterService.java | 21 ++-- .../ClusterNameExpressionResolver.java | 100 ++++++++++++++++++ .../search/RemoteClusterServiceTests.java | 8 +- .../ClusterNameExpressionResolverTests.java | 75 +++++++++++++ .../test/multi_cluster/10_basic.yaml | 24 +++++ 5 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java create mode 100644 core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java diff --git a/core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java b/core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java index bf60b9519fe32..34cb5a84da755 100644 --- a/core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java +++ b/core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -56,6 +57,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -112,11 +114,13 @@ public final class RemoteClusterService extends AbstractComponent implements Clo private final TransportService transportService; private final int numRemoteConnections; + private final ClusterNameExpressionResolver clusterNameResolver; private volatile Map remoteClusters = Collections.emptyMap(); RemoteClusterService(Settings settings, TransportService transportService) { super(settings); this.transportService = transportService; + this.clusterNameResolver = new ClusterNameExpressionResolver(settings); numRemoteConnections = REMOTE_CONNECTIONS_PER_CLUSTER.get(settings); } @@ -204,25 +208,30 @@ boolean isRemoteNodeConnected(final String remoteCluster, final DiscoveryNode no */ Map> groupClusterIndices(String[] requestIndices, Predicate indexExists) { Map> perClusterIndices = new HashMap<>(); + Set remoteClusterNames = this.remoteClusters.keySet(); for (String index : requestIndices) { int i = index.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR); - String indexName = index; - String clusterName = LOCAL_CLUSTER_GROUP_KEY; if (i >= 0) { String remoteClusterName = index.substring(0, i); - if (isRemoteClusterRegistered(remoteClusterName)) { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusterNames, remoteClusterName); + if (clusters.isEmpty() == false) { if (indexExists.test(index)) { // we use : as a separator for remote clusters. might conflict if there is an index that is actually named // remote_cluster_alias:index_name - for this case we fail the request. the user can easily change the cluster alias // if that happens throw new IllegalArgumentException("Can not filter indices; index " + index + " exists but there is also a remote cluster named: " + remoteClusterName); + } + String indexName = index.substring(i + 1); + for (String clusterName : clusters) { + perClusterIndices.computeIfAbsent(clusterName, k -> new ArrayList<>()).add(indexName); } - indexName = index.substring(i + 1); - clusterName = remoteClusterName; + } else { + perClusterIndices.computeIfAbsent(LOCAL_CLUSTER_GROUP_KEY, k -> new ArrayList<>()).add(index); } + } else { + perClusterIndices.computeIfAbsent(LOCAL_CLUSTER_GROUP_KEY, k -> new ArrayList<>()).add(index); } - perClusterIndices.computeIfAbsent(clusterName, k -> new ArrayList()).add(indexName); } return perClusterIndices; } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java new file mode 100644 index 0000000000000..2032c2f4ef3ba --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Resolves cluster names from an expression. The expression must be the exact match of a cluster + * name or must be a wildcard expression. + */ +public final class ClusterNameExpressionResolver extends AbstractComponent { + + private final WildcardExpressionResolver wildcardResolver = new WildcardExpressionResolver(); + + public ClusterNameExpressionResolver(Settings settings) { + super(settings); + } + + /** + * Resolves the provided cluster expression to matching cluster names. This method only + * supports exact or wildcard matches. + * + * @param remoteClusters the aliases for remote clusters + * @param clusterExpression the expressions that can be resolved to cluster names. + * @return the resolved cluster aliases. + */ + public List resolveClusterNames(Set remoteClusters, String clusterExpression) { + if (remoteClusters.contains(clusterExpression)) { + return Collections.singletonList(clusterExpression); + } else if (Regex.isSimpleMatchPattern(clusterExpression)) { + return wildcardResolver.resolve(remoteClusters, clusterExpression); + } else { + return Collections.emptyList(); + } + } + + private static class WildcardExpressionResolver { + + private List resolve(Set remoteClusters, String clusterExpression) { + if (isTrivialWildcard(clusterExpression)) { + return resolveTrivialWildcard(remoteClusters); + } + + Set matches = matches(remoteClusters, clusterExpression); + if (matches.isEmpty()) { + return Collections.emptyList(); + } else { + return new ArrayList<>(matches); + } + } + + private boolean isTrivialWildcard(String clusterExpression) { + return Regex.isMatchAllPattern(clusterExpression); + } + + private List resolveTrivialWildcard(Set remoteClusters) { + return new ArrayList<>(remoteClusters); + } + + private static Set matches(Set remoteClusters, String expression) { + if (expression.indexOf("*") == expression.length() - 1) { + return otherWildcard(remoteClusters, expression); + } else { + return otherWildcard(remoteClusters, expression); + } + } + + private static Set otherWildcard(Set remoteClusters, String expression) { + final String pattern = expression; + return remoteClusters.stream() + .filter(n -> Regex.simpleMatch(pattern, n)) + .collect(Collectors.toSet()); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java b/core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java index d0f0427e71084..81ee9141e2b59 100644 --- a/core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java @@ -143,14 +143,14 @@ public void testGroupClusterIndices() throws IOException { assertTrue(service.isRemoteClusterRegistered("cluster_2")); assertFalse(service.isRemoteClusterRegistered("foo")); Map> perClusterIndices = service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar", - "cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo"}, i -> false); + "cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo", "cluster*:baz", "*:boo", "no*match:boo"}, i -> false); String[] localIndices = perClusterIndices.computeIfAbsent(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, k -> Collections.emptyList()).toArray(new String[0]); assertNotNull(perClusterIndices.remove(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY)); - assertArrayEquals(new String[]{"foo:bar", "foo"}, localIndices); + assertArrayEquals(new String[]{"foo:bar", "foo", "no*match:boo"}, localIndices); assertEquals(2, perClusterIndices.size()); - assertEquals(Arrays.asList("bar", "test"), perClusterIndices.get("cluster_1")); - assertEquals(Arrays.asList("foo:bar", "foo*"), perClusterIndices.get("cluster_2")); + assertEquals(Arrays.asList("bar", "test", "baz", "boo"), perClusterIndices.get("cluster_1")); + assertEquals(Arrays.asList("foo:bar", "foo*", "baz", "boo"), perClusterIndices.get("cluster_2")); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar", diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java new file mode 100644 index 0000000000000..d6c8707c1d76e --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ClusterNameExpressionResolverTests extends ESTestCase { + + private ClusterNameExpressionResolver clusterNameResolver = new ClusterNameExpressionResolver(Settings.EMPTY); + private static final Set remoteClusters = new HashSet<>(); + + static { + remoteClusters.add("cluster1"); + remoteClusters.add("cluster2"); + remoteClusters.add("totallyDifferent"); + } + + public void testExactMatch() { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totallyDifferent"); + assertEquals(new HashSet<>(Arrays.asList("totallyDifferent")), new HashSet<>(clusters)); + } + + public void testNoWildCardNoMatch() { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totallyDifferent2"); + assertTrue(clusters.isEmpty()); + } + + public void testWildCardNoMatch() { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "totally*2"); + assertTrue(clusters.isEmpty()); + } + + public void testSimpleWildCard() { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "*"); + assertEquals(new HashSet<>(Arrays.asList("cluster1", "cluster2", "totallyDifferent")), new HashSet<>(clusters)); + } + + public void testSuffixWildCard() { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "cluster*"); + assertEquals(new HashSet<>(Arrays.asList("cluster1", "cluster2")), new HashSet<>(clusters)); + } + + public void testPrefixWildCard() { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "*Different"); + assertEquals(new HashSet<>(Arrays.asList("totallyDifferent")), new HashSet<>(clusters)); + } + + public void testMiddleWildCard() { + List clusters = clusterNameResolver.resolveClusterNames(remoteClusters, "clu*1"); + assertEquals(new HashSet<>(Arrays.asList("cluster1")), new HashSet<>(clusters)); + } +} diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml index 28ff1e52b876e..bca0703d457bf 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml @@ -118,6 +118,30 @@ - match: { hits.total: 6 } - match: { hits.hits.0._index: "test_remote_cluster:test_index" } +--- +"Test wildcard search": + - do: + cluster.get_settings: + include_defaults: true + + - set: { defaults.search.remote.my_remote_cluster.seeds.0: remote_ip } + + - do: + cluster.put_settings: + flat_settings: true + body: + transient: + search.remote.test_remote_cluster.seeds: $remote_ip + + - match: {transient: {search.remote.test_remote_cluster.seeds: $remote_ip}} + + - do: + search: + index: "*:test_index" + + - match: { _shards.total: 6 } + - match: { hits.total: 12 } + --- "Search an filtered alias on the remote cluster":