From 29c1fcb238e259846145d06fd404d474648f509f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 17 Oct 2021 15:47:47 +0200 Subject: [PATCH] Implement Exclude Patterns for Snapshot- and Repository Names in Get Snapshots API (#77308) (#79315) It's in the title. Adds support for exclude patterns combined with wildcard requests similar to what we support for index names. --- .../apis/get-repo-api.asciidoc | 2 +- .../apis/get-snapshot-api.asciidoc | 89 +++++- .../snapshots/GetSnapshotsIT.java | 297 +++++++++++++++--- .../get/TransportGetRepositoriesAction.java | 49 ++- .../snapshots/get/GetSnapshotsRequest.java | 1 - .../get/TransportGetSnapshotsAction.java | 52 ++- .../rest/action/cat/RestSnapshotAction.java | 6 +- 7 files changed, 403 insertions(+), 93 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc index 7360f550b46b4..68c3b3dd0ec51 100644 --- a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc @@ -46,7 +46,7 @@ GET /_snapshot/my_repository ``:: (Optional, string) Comma-separated list of snapshot repository names used to limit the request. -Wildcard (`*`) expressions are supported. +Wildcard (`*`) expressions are supported including combining wildcards with exclude patterns starting with `-`. + To get information about all snapshot repositories registered in the cluster, omit this parameter or use `*` or `_all`. diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index 60ccf2fec4084..e2c37c1c1731c 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -60,14 +60,15 @@ Use the get snapshot API to return information about one or more snapshots, incl ``:: (Required, string) Comma-separated list of snapshot repository names used to limit the request. -Wildcard (`*`) expressions are supported. +Wildcard (`*`) expressions are supported including combining wildcards with exclude patterns starting with `-`. + To get information about all snapshot repositories registered in the cluster, omit this parameter or use `*` or `_all`. ``:: (Required, string) -Comma-separated list of snapshot names to retrieve. Also accepts wildcards (`*`). +Comma-separated list of snapshot names to retrieve. +Wildcard (`*`) expressions are supported including combining wildcards with exclude patterns starting with `-`. + * To get information about all snapshots in a registered repository, use a wildcard (`*`) or `_all`. * To get information about any snapshots that are currently running, use `_current`. @@ -151,8 +152,8 @@ exclusive with using the `after` parameter. Defaults to `0`. `slm_policy_filter`:: (Optional, string) Filter snapshots by a comma-separated list of SLM policy names that snapshots belong to. Also accepts wildcards (`\*`) and combinations -of wildcards followed by exclude patterns starting in `-`. For example, the pattern `*,-policy-a-\*` will return all snapshots except -for those that were created by an SLM policy with a name starting in `policy-a-`. Note that the wildcard pattern `*` matches all snapshots +of wildcards followed by exclude patterns starting with `-`. For example, the pattern `*,-policy-a-\*` will return all snapshots except +for those that were created by an SLM policy with a name starting with `policy-a-`. Note that the wildcard pattern `*` matches all snapshots created by an SLM policy but not those snapshots that were not created by an SLM policy. To include snapshots not created by an SLM policy you can use the special pattern `_none` that will match all snapshots without an SLM policy. @@ -547,3 +548,83 @@ The API returns the following response: // TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/] // TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/] // TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/] + +The following request returns information for all snapshots with prefix `snapshot` in the `my_repository` repository, +except for the one named `snapshot_3` + +[source,console] +---- +GET /_snapshot/my_repository/snapshot*,-snapshot_3?sort=name +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "snapshots": [ + { + "snapshot": "snapshot_1", + "uuid": "dKb54xw67gvdRctLCxSket", + "repository": "my_repository", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "feature_states": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.129Z", + "start_time_in_millis": 1593093628850, + "end_time": "2020-07-06T21:55:18.129Z", + "end_time_in_millis": 1593094752018, + "duration_in_millis": 0, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + }, + { + "snapshot": "snapshot_2", + "uuid": "vdRctLCxSketdKb54xw67g", + "repository": "my_repository", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "feature_states": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.130Z", + "start_time_in_millis": 1593093628851, + "end_time": "2020-07-06T21:55:18.130Z", + "end_time_in_millis": 1593094752019, + "duration_in_millis": 1, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + } + ], + "total": 2, + "remaining": 0 +} +---- +// TESTRESPONSE[s/"uuid": "dKb54xw67gvdRctLCxSket"/"uuid": $body.snapshots.0.uuid/] +// TESTRESPONSE[s/"uuid": "vdRctLCxSketdKb54xw67g"/"uuid": $body.snapshots.1.uuid/] +// TESTRESPONSE[s/"version_id": /"version_id": $body.snapshots.0.version_id/] +// TESTRESPONSE[s/"version": /"version": $body.snapshots.0.version/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.130Z"/"start_time": $body.snapshots.1.start_time/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628851/"start_time_in_millis": $body.snapshots.1.start_time_in_millis/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.130Z"/"end_time": $body.snapshots.1.end_time/] +// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/] +// TESTRESPONSE[s/"end_time_in_millis": 1593094752019/"end_time_in_millis": $body.snapshots.1.end_time_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 1/"duration_in_millis": $body.snapshots.1.duration_in_millis/] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java index 564f76d178944..fc00facb3e70e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.cluster.repositories.get.TransportGetRepositoriesAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequestBuilder; @@ -25,6 +26,7 @@ import java.util.List; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.is; @@ -75,38 +77,39 @@ public void testSortBy() throws Exception { private void doTestSortOrder(String repoName, Collection allSnapshotNames, SortOrder order) { final List defaultSorting = clusterAdmin().prepareGetSnapshots(repoName).setOrder(order).get().getSnapshots(); assertSnapshotListSorted(defaultSorting, null, order); + final String[] repos = { repoName }; assertSnapshotListSorted( - allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.NAME, order), + allSnapshotsSorted(allSnapshotNames, repos, GetSnapshotsRequest.SortBy.NAME, order), GetSnapshotsRequest.SortBy.NAME, order ); assertSnapshotListSorted( - allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.DURATION, order), + allSnapshotsSorted(allSnapshotNames, repos, GetSnapshotsRequest.SortBy.DURATION, order), GetSnapshotsRequest.SortBy.DURATION, order ); assertSnapshotListSorted( - allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.INDICES, order), + allSnapshotsSorted(allSnapshotNames, repos, GetSnapshotsRequest.SortBy.INDICES, order), GetSnapshotsRequest.SortBy.INDICES, order ); assertSnapshotListSorted( - allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.START_TIME, order), + allSnapshotsSorted(allSnapshotNames, repos, GetSnapshotsRequest.SortBy.START_TIME, order), GetSnapshotsRequest.SortBy.START_TIME, order ); assertSnapshotListSorted( - allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.SHARDS, order), + allSnapshotsSorted(allSnapshotNames, repos, GetSnapshotsRequest.SortBy.SHARDS, order), GetSnapshotsRequest.SortBy.SHARDS, order ); assertSnapshotListSorted( - allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.FAILED_SHARDS, order), + allSnapshotsSorted(allSnapshotNames, repos, GetSnapshotsRequest.SortBy.FAILED_SHARDS, order), GetSnapshotsRequest.SortBy.FAILED_SHARDS, order ); assertSnapshotListSorted( - allSnapshotsSorted(allSnapshotNames, repoName, GetSnapshotsRequest.SortBy.REPOSITORY, order), + allSnapshotsSorted(allSnapshotNames, repos, GetSnapshotsRequest.SortBy.REPOSITORY, order), GetSnapshotsRequest.SortBy.REPOSITORY, order ); @@ -127,22 +130,23 @@ public void testResponseSizeLimit() throws Exception { } private void doTestPagination(String repoName, List names, GetSnapshotsRequest.SortBy sort, SortOrder order) { - final List allSnapshotsSorted = allSnapshotsSorted(names, repoName, sort, order); - final GetSnapshotsResponse batch1 = sortedWithLimit(repoName, sort, null, 2, order); + final String[] repos = { repoName }; + final List allSnapshotsSorted = allSnapshotsSorted(names, repos, sort, order); + final GetSnapshotsResponse batch1 = sortedWithLimit(repos, sort, null, 2, order); assertEquals(allSnapshotsSorted.subList(0, 2), batch1.getSnapshots()); - final GetSnapshotsResponse batch2 = sortedWithLimit(repoName, sort, batch1.next(), 2, order); + final GetSnapshotsResponse batch2 = sortedWithLimit(repos, sort, batch1.next(), 2, order); assertEquals(allSnapshotsSorted.subList(2, 4), batch2.getSnapshots()); final int lastBatch = names.size() - batch1.getSnapshots().size() - batch2.getSnapshots().size(); - final GetSnapshotsResponse batch3 = sortedWithLimit(repoName, sort, batch2.next(), lastBatch, order); + final GetSnapshotsResponse batch3 = sortedWithLimit(repos, sort, batch2.next(), lastBatch, order); assertEquals( batch3.getSnapshots(), allSnapshotsSorted.subList(batch1.getSnapshots().size() + batch2.getSnapshots().size(), names.size()) ); - final GetSnapshotsResponse batch3NoLimit = sortedWithLimit(repoName, sort, batch2.next(), GetSnapshotsRequest.NO_LIMIT, order); + final GetSnapshotsResponse batch3NoLimit = sortedWithLimit(repos, sort, batch2.next(), GetSnapshotsRequest.NO_LIMIT, order); assertNull(batch3NoLimit.next()); assertEquals(batch3.getSnapshots(), batch3NoLimit.getSnapshots()); final GetSnapshotsResponse batch3LargeLimit = sortedWithLimit( - repoName, + repos, sort, batch2.next(), lastBatch + randomIntBetween(1, 100), @@ -172,18 +176,27 @@ public void testSortAndPaginateWithInProgress() throws Exception { } awaitNumberOfSnapshotsInProgress(inProgressCount); - assertStablePagination(repoName, allSnapshotNames, GetSnapshotsRequest.SortBy.START_TIME); - assertStablePagination(repoName, allSnapshotNames, GetSnapshotsRequest.SortBy.NAME); - assertStablePagination(repoName, allSnapshotNames, GetSnapshotsRequest.SortBy.INDICES); + final String[] repos = { repoName }; + assertStablePagination(repos, allSnapshotNames, GetSnapshotsRequest.SortBy.START_TIME); + assertStablePagination(repos, allSnapshotNames, GetSnapshotsRequest.SortBy.NAME); + assertStablePagination(repos, allSnapshotNames, GetSnapshotsRequest.SortBy.INDICES); + + assertThat( + clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(GetSnapshotsRequest.CURRENT_SNAPSHOT, "-snap*") + .get() + .getSnapshots(), + empty() + ); unblockAllDataNodes(repoName); for (ActionFuture inProgressSnapshot : inProgressSnapshots) { assertSuccessful(inProgressSnapshot); } - assertStablePagination(repoName, allSnapshotNames, GetSnapshotsRequest.SortBy.START_TIME); - assertStablePagination(repoName, allSnapshotNames, GetSnapshotsRequest.SortBy.NAME); - assertStablePagination(repoName, allSnapshotNames, GetSnapshotsRequest.SortBy.INDICES); + assertStablePagination(repos, allSnapshotNames, GetSnapshotsRequest.SortBy.START_TIME); + assertStablePagination(repos, allSnapshotNames, GetSnapshotsRequest.SortBy.NAME); + assertStablePagination(repos, allSnapshotNames, GetSnapshotsRequest.SortBy.INDICES); } public void testPaginationRequiresVerboseListing() throws Exception { @@ -210,12 +223,190 @@ public void testPaginationRequiresVerboseListing() throws Exception { ); } + public void testExcludePatterns() throws Exception { + final String repoName1 = "test-repo-1"; + final String repoName2 = "test-repo-2"; + final String otherRepo = "other-repo"; + createRepository(repoName1, "fs"); + createRepository(repoName2, "fs"); + createRepository(otherRepo, "fs"); + + final List namesRepo1 = createNSnapshots(repoName1, randomIntBetween(1, 5)); + final List namesRepo2 = createNSnapshots(repoName2, randomIntBetween(1, 5)); + final List namesOtherRepo = createNSnapshots(otherRepo, randomIntBetween(1, 5)); + + final Collection allSnapshotNames = new HashSet<>(namesRepo1); + allSnapshotNames.addAll(namesRepo2); + final Collection allSnapshotNamesWithoutOther = org.elasticsearch.core.Set.copyOf(allSnapshotNames); + allSnapshotNames.addAll(namesOtherRepo); + + final SortOrder order = SortOrder.DESC; + final List allSorted = allSnapshotsSorted( + allSnapshotNames, + new String[] { "*" }, + GetSnapshotsRequest.SortBy.REPOSITORY, + order + ); + final List allSortedWithoutOther = allSnapshotsSorted( + allSnapshotNamesWithoutOther, + new String[] { "*", "-" + otherRepo }, + GetSnapshotsRequest.SortBy.REPOSITORY, + order + ); + assertThat(allSortedWithoutOther, is(allSorted.subList(0, allSnapshotNamesWithoutOther.size()))); + + final List allInOther = allSnapshotsSorted( + namesOtherRepo, + new String[] { "*", "-test-repo-*" }, + GetSnapshotsRequest.SortBy.REPOSITORY, + order + ); + assertThat(allInOther, is(allSorted.subList(allSnapshotNamesWithoutOther.size(), allSorted.size()))); + + final String otherPrefixSnapshot1 = "other-prefix-snapshot-1"; + createFullSnapshot(otherRepo, otherPrefixSnapshot1); + final String otherPrefixSnapshot2 = "other-prefix-snapshot-2"; + createFullSnapshot(otherRepo, otherPrefixSnapshot2); + + final String[] patternOtherRepo = randomBoolean() ? new String[] { otherRepo } : new String[] { "*", "-test-repo-*" }; + final List allInOtherWithoutOtherPrefix = allSnapshotsSorted( + namesOtherRepo, + patternOtherRepo, + GetSnapshotsRequest.SortBy.REPOSITORY, + order, + "-other*" + ); + assertThat(allInOtherWithoutOtherPrefix, is(allInOther)); + + final List allInOtherWithoutOtherExplicit = allSnapshotsSorted( + namesOtherRepo, + patternOtherRepo, + GetSnapshotsRequest.SortBy.REPOSITORY, + order, + "-" + otherPrefixSnapshot1, + "-" + otherPrefixSnapshot2 + ); + assertThat(allInOtherWithoutOtherExplicit, is(allInOther)); + + assertThat(clusterAdmin().prepareGetSnapshots(matchAllPattern()).setSnapshots("other*", "-o*").get().getSnapshots(), empty()); + assertThat(clusterAdmin().prepareGetSnapshots("other*", "-o*").setSnapshots(matchAllPattern()).get().getSnapshots(), empty()); + assertThat( + clusterAdmin().prepareGetSnapshots("other*", otherRepo, "-o*").setSnapshots(matchAllPattern()).get().getSnapshots(), + empty() + ); + assertThat( + clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots("non-existing*", otherPrefixSnapshot1, "-o*") + .get() + .getSnapshots(), + empty() + ); + } + + public void testNamesStartingInDash() { + final String repoName1 = "test-repo"; + final String weirdRepo1 = "-weird-repo-1"; + final String weirdRepo2 = "-weird-repo-2"; + createRepository(repoName1, "fs"); + createRepository(weirdRepo1, "fs"); + createRepository(weirdRepo2, "fs"); + + final String snapshotName = "test-snapshot"; + final String weirdSnapshot1 = "-weird-snapshot-1"; + final String weirdSnapshot2 = "-weird-snapshot-2"; + + final SnapshotInfo snapshotInRepo1 = createFullSnapshot(repoName1, snapshotName); + final SnapshotInfo weirdSnapshot1InRepo1 = createFullSnapshot(repoName1, weirdSnapshot1); + final SnapshotInfo weirdSnapshot2InRepo1 = createFullSnapshot(repoName1, weirdSnapshot2); + + final SnapshotInfo snapshotInWeird1 = createFullSnapshot(weirdRepo1, snapshotName); + final SnapshotInfo weirdSnapshot1InWeird1 = createFullSnapshot(weirdRepo1, weirdSnapshot1); + final SnapshotInfo weirdSnapshot2InWeird1 = createFullSnapshot(weirdRepo1, weirdSnapshot2); + + final SnapshotInfo snapshotInWeird2 = createFullSnapshot(weirdRepo2, snapshotName); + final SnapshotInfo weirdSnapshot1InWeird2 = createFullSnapshot(weirdRepo2, weirdSnapshot1); + final SnapshotInfo weirdSnapshot2InWeird2 = createFullSnapshot(weirdRepo2, weirdSnapshot2); + + final List allSnapshots = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSort(GetSnapshotsRequest.SortBy.REPOSITORY) + .get() + .getSnapshots(); + assertThat(allSnapshots, hasSize(9)); + + final List allSnapshotsByAll = getAllByPatterns(matchAllPattern(), matchAllPattern()); + assertThat(allSnapshotsByAll, is(allSnapshots)); + assertThat(getAllByPatterns(matchAllPattern(), new String[] { snapshotName, weirdSnapshot1, weirdSnapshot2 }), is(allSnapshots)); + assertThat(getAllByPatterns(new String[] { repoName1, weirdRepo1, weirdRepo2 }, matchAllPattern()), is(allSnapshots)); + + assertThat( + getAllByPatterns(matchAllPattern(), new String[] { snapshotName }), + is(org.elasticsearch.core.List.of(snapshotInWeird1, snapshotInWeird2, snapshotInRepo1)) + ); + assertThat( + getAllByPatterns(matchAllPattern(), new String[] { weirdSnapshot1 }), + is(org.elasticsearch.core.List.of(weirdSnapshot1InWeird1, weirdSnapshot1InWeird2, weirdSnapshot1InRepo1)) + ); + assertThat( + getAllByPatterns(matchAllPattern(), new String[] { snapshotName, weirdSnapshot1 }), + is( + org.elasticsearch.core.List.of( + weirdSnapshot1InWeird1, + snapshotInWeird1, + weirdSnapshot1InWeird2, + snapshotInWeird2, + weirdSnapshot1InRepo1, + snapshotInRepo1 + ) + ) + ); + assertThat(getAllByPatterns(matchAllPattern(), new String[] { "non-existing*", weirdSnapshot1 }), empty()); + assertThat( + getAllByPatterns(matchAllPattern(), new String[] { "*", "--weird-snapshot-1" }), + is( + org.elasticsearch.core.List.of( + weirdSnapshot2InWeird1, + snapshotInWeird1, + weirdSnapshot2InWeird2, + snapshotInWeird2, + weirdSnapshot2InRepo1, + snapshotInRepo1 + ) + ) + ); + assertThat( + getAllByPatterns(matchAllPattern(), new String[] { "-*" }), + is( + org.elasticsearch.core.List.of( + weirdSnapshot1InWeird1, + weirdSnapshot2InWeird1, + weirdSnapshot1InWeird2, + weirdSnapshot2InWeird2, + weirdSnapshot1InRepo1, + weirdSnapshot2InRepo1 + + ) + ) + ); + } + + private static String[] matchAllPattern() { + return randomBoolean() ? new String[] { "*" } : new String[] { TransportGetRepositoriesAction.ALL_PATTERN }; + } + + private List getAllByPatterns(String[] repos, String[] snapshots) { + return clusterAdmin().prepareGetSnapshots(repos) + .setSnapshots(snapshots) + .setSort(GetSnapshotsRequest.SortBy.REPOSITORY) + .get() + .getSnapshots(); + } + public void testFilterBySLMPolicy() throws Exception { final String repoName = "test-repo"; createRepository(repoName, "fs"); createNSnapshots(repoName, randomIntBetween(1, 5)); - final List snapshotsWithoutPolicy = clusterAdmin().prepareGetSnapshots("*") - .setSnapshots("*") + final List snapshotsWithoutPolicy = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) .setSort(GetSnapshotsRequest.SortBy.NAME) .get() .getSnapshots(); @@ -256,8 +447,8 @@ public void testFilterBySLMPolicy() throws Exception { is(org.elasticsearch.core.List.of(withOtherPolicy, withPolicy)) ); - final List allSnapshots = clusterAdmin().prepareGetSnapshots("*") - .setSnapshots("*") + final List allSnapshots = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) .setSort(GetSnapshotsRequest.SortBy.NAME) .get() .getSnapshots(); @@ -266,20 +457,20 @@ public void testFilterBySLMPolicy() throws Exception { } private static List getAllSnapshotsForPolicies(String... policies) { - return clusterAdmin().prepareGetSnapshots("*") - .setSnapshots("*") + return clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) .setPolicies(policies) .setSort(GetSnapshotsRequest.SortBy.NAME) .get() .getSnapshots(); } - private static void assertStablePagination(String repoName, Collection allSnapshotNames, GetSnapshotsRequest.SortBy sort) { + private static void assertStablePagination(String[] repoNames, Collection allSnapshotNames, GetSnapshotsRequest.SortBy sort) { final SortOrder order = randomFrom(SortOrder.values()); - final List allSorted = allSnapshotsSorted(allSnapshotNames, repoName, sort, order); + final List allSorted = allSnapshotsSorted(allSnapshotNames, repoNames, sort, order); for (int i = 1; i <= allSnapshotNames.size(); i++) { - final GetSnapshotsResponse subsetSorted = sortedWithLimit(repoName, sort, null, i, order); + final GetSnapshotsResponse subsetSorted = sortedWithLimit(repoNames, sort, null, i, order); assertEquals(allSorted.subList(0, i), subsetSorted.getSnapshots()); } @@ -287,13 +478,13 @@ private static void assertStablePagination(String repoName, Collection a final SnapshotInfo after = allSorted.get(j); for (int i = 1; i < allSnapshotNames.size() - j; i++) { final GetSnapshotsResponse getSnapshotsResponse = sortedWithLimit( - repoName, + repoNames, sort, GetSnapshotsRequest.After.from(after, sort).asQueryParam(), i, order ); - final GetSnapshotsResponse getSnapshotsResponseNumeric = sortedWithLimit(repoName, sort, j + 1, i, order); + final GetSnapshotsResponse getSnapshotsResponseNumeric = sortedWithLimit(repoNames, sort, j + 1, i, order); final List subsetSorted = getSnapshotsResponse.getSnapshots(); assertEquals(subsetSorted, getSnapshotsResponseNumeric.getSnapshots()); assertEquals(subsetSorted, allSorted.subList(j + 1, j + i + 1)); @@ -308,11 +499,19 @@ private static void assertStablePagination(String repoName, Collection a private static List allSnapshotsSorted( Collection allSnapshotNames, - String repoName, + String[] repoNames, GetSnapshotsRequest.SortBy sortBy, - SortOrder order + SortOrder order, + String... namePatterns ) { - final GetSnapshotsResponse getSnapshotsResponse = sortedWithLimit(repoName, sortBy, null, GetSnapshotsRequest.NO_LIMIT, order); + final GetSnapshotsResponse getSnapshotsResponse = sortedWithLimit( + repoNames, + sortBy, + null, + GetSnapshotsRequest.NO_LIMIT, + order, + namePatterns + ); final List snapshotInfos = getSnapshotsResponse.getSnapshots(); assertEquals(snapshotInfos.size(), allSnapshotNames.size()); assertEquals(getSnapshotsResponse.totalCount(), allSnapshotNames.size()); @@ -324,37 +523,33 @@ private static List allSnapshotsSorted( } private static GetSnapshotsResponse sortedWithLimit( - String repoName, + String[] repoNames, GetSnapshotsRequest.SortBy sortBy, String after, int size, - SortOrder order + SortOrder order, + String... namePatterns ) { - return baseGetSnapshotsRequest(repoName).setAfter(after).setSort(sortBy).setSize(size).setOrder(order).get(); + return baseGetSnapshotsRequest(repoNames).setAfter(after) + .setSort(sortBy) + .setSize(size) + .setOrder(order) + .addSnapshots(namePatterns) + .get(); } private static GetSnapshotsResponse sortedWithLimit( - String repoName, + String[] repoNames, GetSnapshotsRequest.SortBy sortBy, int offset, int size, SortOrder order ) { - return baseGetSnapshotsRequest(repoName).setOffset(offset).setSort(sortBy).setSize(size).setOrder(order).get(); + return baseGetSnapshotsRequest(repoNames).setOffset(offset).setSort(sortBy).setSize(size).setOrder(order).get(); } - private static GetSnapshotsRequestBuilder baseGetSnapshotsRequest(String repoName) { - final GetSnapshotsRequestBuilder builder = clusterAdmin().prepareGetSnapshots(repoName); - // exclude old version snapshot from test assertions every time and do a prefixed query in either case half the time - if (randomBoolean() - || clusterAdmin().prepareGetSnapshots(repoName) - .setSnapshots(AbstractSnapshotIntegTestCase.OLD_VERSION_SNAPSHOT_PREFIX + "*") - .setIgnoreUnavailable(true) - .get() - .getSnapshots() - .isEmpty() == false) { - builder.setSnapshots(RANDOM_SNAPSHOT_NAME_PREFIX + "*"); - } - return builder; + private static GetSnapshotsRequestBuilder baseGetSnapshotsRequest(String[] repoNames) { + return clusterAdmin().prepareGetSnapshots(repoNames) + .setSnapshots("*", "-" + AbstractSnapshotIntegTestCase.OLD_VERSION_SNAPSHOT_PREFIX + "*"); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java index 3cb65c6b794a7..d4edb4dffb38c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.repositories.RepositoryMissingException; @@ -34,6 +35,8 @@ */ public class TransportGetRepositoriesAction extends TransportMasterNodeReadAction { + public static final String ALL_PATTERN = "_all"; + @Inject public TransportGetRepositoriesAction( TransportService transportService, @@ -55,6 +58,11 @@ public TransportGetRepositoriesAction( ); } + public static boolean isMatchAll(String[] patterns) { + return (patterns.length == 0) + || (patterns.length == 1 && (ALL_PATTERN.equalsIgnoreCase(patterns[0]) || Regex.isMatchAllPattern(patterns[0]))); + } + @Override protected ClusterBlockException checkBlock(GetRepositoriesRequest request, ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); @@ -78,30 +86,37 @@ protected void masterOperation( */ public static List getRepositories(ClusterState state, String[] repoNames) { RepositoriesMetadata repositories = state.metadata().custom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY); - if (repoNames.length == 0 || (repoNames.length == 1 && ("_all".equals(repoNames[0]) || "*".equals(repoNames[0])))) { + if (isMatchAll(repoNames)) { return repositories.repositories(); - } else { - Set repositoriesToGet = new LinkedHashSet<>(); // to keep insertion order - for (String repositoryOrPattern : repoNames) { - if (Regex.isSimpleMatchPattern(repositoryOrPattern) == false) { - repositoriesToGet.add(repositoryOrPattern); + } + final List includePatterns = new ArrayList<>(); + final List excludePatterns = new ArrayList<>(); + boolean seenWildcard = false; + for (String repositoryOrPattern : repoNames) { + if (seenWildcard && repositoryOrPattern.length() > 1 && repositoryOrPattern.startsWith("-")) { + excludePatterns.add(repositoryOrPattern.substring(1)); + } else { + if (Regex.isSimpleMatchPattern(repositoryOrPattern)) { + seenWildcard = true; } else { - for (RepositoryMetadata repository : repositories.repositories()) { - if (Regex.simpleMatch(repositoryOrPattern, repository.name())) { - repositoriesToGet.add(repository.name()); - } + if (repositories.repository(repositoryOrPattern) == null) { + throw new RepositoryMissingException(repositoryOrPattern); } } + includePatterns.add(repositoryOrPattern); } - List repositoryListBuilder = new ArrayList<>(); - for (String repository : repositoriesToGet) { - RepositoryMetadata repositoryMetadata = repositories.repository(repository); - if (repositoryMetadata == null) { - throw new RepositoryMissingException(repository); + } + final String[] excludes = excludePatterns.toArray(Strings.EMPTY_ARRAY); + final Set repositoryListBuilder = new LinkedHashSet<>(); // to keep insertion order + for (String repositoryOrPattern : includePatterns) { + for (RepositoryMetadata repository : repositories.repositories()) { + if (repositoryListBuilder.contains(repository) == false + && Regex.simpleMatch(repositoryOrPattern, repository.name()) + && Regex.simpleMatch(excludes, repository.name()) == false) { + repositoryListBuilder.add(repository); } - repositoryListBuilder.add(repositoryMetadata); } - return repositoryListBuilder; } + return org.elasticsearch.core.List.copyOf(repositoryListBuilder); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index ba701222c0ab9..46581ed3fd2cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -36,7 +36,6 @@ */ public class GetSnapshotsRequest extends MasterNodeRequest { - public static final String ALL_SNAPSHOTS = "_all"; public static final String CURRENT_SNAPSHOT = "_current"; public static final String NO_POLICY_PATTERN = "_none"; public static final boolean DEFAULT_VERBOSE_MODE = true; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 7fd930299b69c..b2649a5575b6b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -299,27 +299,49 @@ private void loadSnapshotInfos( } final Set toResolve = new HashSet<>(); - if (isAllSnapshots(snapshots)) { + if (TransportGetRepositoriesAction.isMatchAll(snapshots)) { toResolve.addAll(allSnapshotIds.values()); } else { + final List includePatterns = new ArrayList<>(); + final List excludePatterns = new ArrayList<>(); + boolean hasCurrent = false; + boolean seenWildcard = false; for (String snapshotOrPattern : snapshots) { - if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) { - toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshot).collect(Collectors.toList())); - } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) { - if (allSnapshotIds.containsKey(snapshotOrPattern)) { - toResolve.add(allSnapshotIds.get(snapshotOrPattern)); - } else if (ignoreUnavailable == false) { - throw new SnapshotMissingException(repo, snapshotOrPattern); - } + if (seenWildcard && snapshotOrPattern.length() > 1 && snapshotOrPattern.startsWith("-")) { + excludePatterns.add(snapshotOrPattern.substring(1)); } else { - for (Map.Entry entry : allSnapshotIds.entrySet()) { - if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) { - toResolve.add(entry.getValue()); + if (Regex.isSimpleMatchPattern(snapshotOrPattern)) { + seenWildcard = true; + includePatterns.add(snapshotOrPattern); + } else if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) { + hasCurrent = true; + seenWildcard = true; + } else { + if (ignoreUnavailable == false && allSnapshotIds.containsKey(snapshotOrPattern) == false) { + throw new SnapshotMissingException(repo, snapshotOrPattern); } + includePatterns.add(snapshotOrPattern); + } + } + } + final String[] includes = includePatterns.toArray(Strings.EMPTY_ARRAY); + final String[] excludes = excludePatterns.toArray(Strings.EMPTY_ARRAY); + for (Map.Entry entry : allSnapshotIds.entrySet()) { + final Snapshot snapshot = entry.getValue(); + if (toResolve.contains(snapshot) == false + && Regex.simpleMatch(includes, entry.getKey()) + && Regex.simpleMatch(excludes, entry.getKey()) == false) { + toResolve.add(snapshot); + } + } + if (hasCurrent) { + for (SnapshotInfo snapshotInfo : currentSnapshots) { + final Snapshot snapshot = snapshotInfo.snapshot(); + if (Regex.simpleMatch(excludes, snapshot.getSnapshotId().getName()) == false) { + toResolve.add(snapshot); } } } - if (toResolve.isEmpty() && ignoreUnavailable == false && isCurrentSnapshotsOnly(snapshots) == false) { throw new SnapshotMissingException(repo, snapshots[0]); } @@ -431,10 +453,6 @@ private void snapshots( ); } - private boolean isAllSnapshots(String[] snapshots) { - return (snapshots.length == 0) || (snapshots.length == 1 && GetSnapshotsRequest.ALL_SNAPSHOTS.equalsIgnoreCase(snapshots[0])); - } - private boolean isCurrentSnapshotsOnly(String[] snapshots) { return (snapshots.length == 1 && GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshots[0])); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java index 009f28713b9bf..929f187eba48e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.cluster.repositories.get.TransportGetRepositoriesAction; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.client.node.NodeClient; @@ -51,9 +52,10 @@ public String getName() { @Override protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient client) { + final String[] matchAll = {TransportGetRepositoriesAction.ALL_PATTERN}; GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest() - .repositories(request.paramAsStringArray("repository", new String[]{"_all"})) - .snapshots(new String[]{GetSnapshotsRequest.ALL_SNAPSHOTS}); + .repositories(request.paramAsStringArray("repository", matchAll)) + .snapshots(matchAll); getSnapshotsRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", getSnapshotsRequest.ignoreUnavailable()));