diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 216241fbde980..9b33c6da5f6d7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -58,12 +58,8 @@ protected ClusterBlockException checkBlock(GetAliasesRequest request, ClusterSta @Override protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener listener) { - String[] concreteIndices; - // Switch to a context which will drop any deprecation warnings, because there may be indices resolved here which are not - // returned in the final response. We'll add warnings back later if necessary in checkSystemIndexAccess. - try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().newStoredContext(false)) { - concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); - } + // resolve all concrete indices upfront and warn/error later + final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(state, request); final SystemIndexAccessLevel systemIndexAccessLevel = indexNameExpressionResolver.getSystemIndexAccessLevel(); ImmutableOpenMap> aliases = state.metadata().findAliases(request, concreteIndices); listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases, state, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java index e5aec68a91dec..d5e1919dc7139 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import java.util.ArrayList; import java.util.Arrays; @@ -63,7 +64,8 @@ public List resolveIndexAbstractions(Iterable indices, IndicesOp // continue indexAbstraction = dateMathName; } else if (availableIndexAbstractions.contains(dateMathName) && - isIndexVisible(indexAbstraction, dateMathName, indicesOptions, metadata, includeDataStreams, true)) { + isIndexVisible(indexAbstraction, dateMathName, indicesOptions, metadata, indexNameExpressionResolver, + includeDataStreams, true)) { if (minus) { finalIndices.remove(dateMathName); } else { @@ -81,7 +83,8 @@ public List resolveIndexAbstractions(Iterable indices, IndicesOp Set resolvedIndices = new HashSet<>(); for (String authorizedIndex : availableIndexAbstractions) { if (Regex.simpleMatch(indexAbstraction, authorizedIndex) && - isIndexVisible(indexAbstraction, authorizedIndex, indicesOptions, metadata, includeDataStreams)) { + isIndexVisible(indexAbstraction, authorizedIndex, indicesOptions, metadata, indexNameExpressionResolver, + includeDataStreams)) { resolvedIndices.add(authorizedIndex); } } @@ -109,12 +112,12 @@ public List resolveIndexAbstractions(Iterable indices, IndicesOp } public static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata, - boolean includeDataStreams) { - return isIndexVisible(expression, index, indicesOptions, metadata, includeDataStreams, false); + IndexNameExpressionResolver resolver, boolean includeDataStreams) { + return isIndexVisible(expression, index, indicesOptions, metadata, resolver, includeDataStreams, false); } public static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, Metadata metadata, - boolean includeDataStreams, boolean dateMathExpression) { + IndexNameExpressionResolver resolver, boolean includeDataStreams, boolean dateMathExpression) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(index); if (indexAbstraction == null) { throw new IllegalStateException("could not resolve index abstraction [" + index + "]"); @@ -127,7 +130,22 @@ public static boolean isIndexVisible(String expression, String index, IndicesOpt return isVisible && indicesOptions.ignoreAliases() == false; } if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { - return isVisible && includeDataStreams; + if (includeDataStreams == false) { + return false; + } + + if (indexAbstraction.isSystem()) { + final SystemIndexAccessLevel level = resolver.getSystemIndexAccessLevel(); + if (level == SystemIndexAccessLevel.ALL) { + return true; + } else if (level == SystemIndexAccessLevel.NONE) { + return false; + } else if (level == SystemIndexAccessLevel.RESTRICTED) { + return resolver.getSystemIndexAccessPredicate().test(indexAbstraction.getName()); + } + } else { + return isVisible; + } } assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index"; // since it is a date math expression, we consider the index visible regardless of open/closed/hidden as the user is using @@ -139,6 +157,22 @@ public static boolean isIndexVisible(String expression, String index, IndicesOpt if (isVisible == false) { return false; } + if (indexAbstraction.isSystem()) { + // system index that backs system data stream + if (indexAbstraction.getParentDataStream() != null) { + if (indexAbstraction.getParentDataStream().isSystem() == false) { + throw new IllegalStateException("system index is part of a data stream that is not a system data stream"); + } + final SystemIndexAccessLevel level = resolver.getSystemIndexAccessLevel(); + if (level == SystemIndexAccessLevel.ALL) { + return true; + } else if (level == SystemIndexAccessLevel.NONE) { + return false; + } else if (level == SystemIndexAccessLevel.RESTRICTED) { + return resolver.getSystemIndexAccessPredicate().test(indexAbstraction.getName()); + } + } + } IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0); if (indexMetadata.getState() == IndexMetadata.State.CLOSE && indicesOptions.expandWildcardsClosed()) { diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java new file mode 100644 index 0000000000000..66df3744e25fa --- /dev/null +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.fleet; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.util.Collections; +import java.util.List; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class FleetDataStreamIT extends ESRestTestCase { + + static final String BASIC_AUTH_VALUE = basicAuthHeaderValue( + "x_pack_rest_user", + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING + ); + + @Override + protected Settings restClientSettings() { + // Note that we are superuser here but DO NOT provide a product origin + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + + @Override + protected Settings restAdminSettings() { + // Note that we are both superuser here and provide a product origin + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE) + .put(ThreadContext.PREFIX + ".X-elastic-product-origin", "fleet") + .build(); + } + + public void testAliasWithSystemDataStream() throws Exception { + // Create a system data stream + Request initialDocResponse = new Request("POST", ".fleet-actions-results/_doc"); + initialDocResponse.setJsonEntity("{\"@timestamp\": 0}"); + assertOK(adminClient().performRequest(initialDocResponse)); + + // Create a system index - this one has an alias + Request sysIdxRequest = new Request("PUT", ".fleet-artifacts"); + assertOK(adminClient().performRequest(sysIdxRequest)); + + // Create a regular index + String regularIndex = "regular-idx"; + String regularAlias = "regular-alias"; + Request regularIdxRequest = new Request("PUT", regularIndex); + regularIdxRequest.setJsonEntity("{\"aliases\": {\"" + regularAlias + "\": {}}}"); + assertOK(client().performRequest(regularIdxRequest)); + + assertGetAliasAPIBehavesAsExpected(regularIndex, regularAlias); + } + + public void testAliasWithSystemIndices() throws Exception { + // Create a system index - this one has an alias + Request sysIdxRequest = new Request("PUT", ".fleet-artifacts"); + assertOK(adminClient().performRequest(sysIdxRequest)); + + // Create a regular index + String regularIndex = "regular-idx"; + String regularAlias = "regular-alias"; + Request regularIdxRequest = new Request("PUT", regularIndex); + regularIdxRequest.setJsonEntity("{\"aliases\": {\"" + regularAlias + "\": {}}}"); + assertOK(client().performRequest(regularIdxRequest)); + + assertGetAliasAPIBehavesAsExpected(regularIndex, regularAlias); + } + + private void assertGetAliasAPIBehavesAsExpected(String regularIndex, String regularAlias) throws Exception { + // Get a non-system alias, should not warn or error + { + Request request = new Request("GET", "_alias/" + regularAlias); + Response response = client().performRequest(request); + assertOK(response); + assertThat( + EntityUtils.toString(response.getEntity()), + allOf(containsString(regularAlias), containsString(regularIndex), not(containsString(".fleet-artifacts"))) + ); + } + + // Fully specify a regular index and alias, should not warn or error + { + Request request = new Request("GET", regularIndex + "/_alias/" + regularAlias); + Response response = client().performRequest(request); + assertOK(response); + assertThat( + EntityUtils.toString(response.getEntity()), + allOf(containsString(regularAlias), containsString(regularIndex), not(containsString(".fleet-artifacts"))) + ); + } + + // The rest of these produce a warning + RequestOptions consumeWarningsOptions = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler( + warnings -> Collections.singletonList( + "this request accesses system indices: [.fleet-artifacts-7], but " + + "in a future major version, direct access to system indices will be prevented by default" + ).equals(warnings) == false + ) + .build(); + + // The base _alias route warns because there is a system index in the response + { + Request request = new Request("GET", "_alias"); + request.setOptions(consumeWarningsOptions); // The result includes system indices, so we warn + Response response = client().performRequest(request); + assertOK(response); + assertThat( + EntityUtils.toString(response.getEntity()), + allOf(containsString(regularAlias), containsString(regularIndex), not(containsString(".fleet-actions-results"))) + ); + } + + // Specify a system alias, should warn + { + Request request = new Request("GET", "_alias/.fleet-artifacts"); + request.setOptions(consumeWarningsOptions); + Response response = client().performRequest(request); + assertOK(response); + assertThat( + EntityUtils.toString(response.getEntity()), + allOf( + containsString(".fleet-artifacts"), + containsString(".fleet-artifacts-7"), + not(containsString(regularAlias)), + not(containsString(regularIndex)) + ) + ); + } + + // Fully specify a system index and alias, should warn + { + Request request = new Request("GET", ".fleet-artifacts-7/_alias/.fleet-artifacts"); + request.setOptions(consumeWarningsOptions); + Response response = client().performRequest(request); + assertOK(response); + assertThat( + EntityUtils.toString(response.getEntity()), + allOf( + containsString(".fleet-artifacts"), + containsString(".fleet-artifacts-7"), + not(containsString(regularAlias)), + not(containsString(regularIndex)) + ) + ); + } + + // Check an alias that doesn't exist + { + Request getAliasRequest = new Request("GET", "_alias/auditbeat-7.13.0"); + try { + client().performRequest(getAliasRequest); + fail("this request should not succeed, as it is looking for an alias that does not exist"); + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(404)); + assertThat( + EntityUtils.toString(e.getResponse().getEntity()), + not(containsString("use and access is reserved for system operations")) + ); + } + } + + // Specify a system data stream as an alias - should 404 + { + Request getAliasRequest = new Request("GET", "_alias/.fleet-actions-results"); + try { + client().performRequest(getAliasRequest); + fail("this request should not succeed, as it is looking for an alias that does not exist"); + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(404)); + assertThat( + EntityUtils.toString(e.getResponse().getEntity()), + not(containsString("use and access is reserved for system operations")) + ); + } + } + } + + public void testCountWithSystemDataStream() throws Exception { + assertThatAPIWildcardResolutionWorks(); + + // Create a system data stream + Request initialDocResponse = new Request("POST", ".fleet-actions-results/_doc"); + initialDocResponse.setJsonEntity("{\"@timestamp\": 0}"); + assertOK(adminClient().performRequest(initialDocResponse)); + assertThatAPIWildcardResolutionWorks(); + + // Create a system index - this one has an alias + Request sysIdxRequest = new Request("PUT", ".fleet-artifacts"); + assertOK(adminClient().performRequest(sysIdxRequest)); + assertThatAPIWildcardResolutionWorks( + singletonList( + "this request accesses system indices: [.fleet-artifacts-7], but in a future major version, direct access to system" + + " indices will be prevented by default" + ) + ); + assertThatAPIWildcardResolutionWorks( + singletonList( + "this request accesses system indices: [.fleet-artifacts-7], but in a future major version, direct access to system" + + " indices will be prevented by default" + ), + ".f*" + ); + + // Create a regular index + String regularIndex = "regular-idx"; + String regularAlias = "regular-alias"; + Request regularIdxRequest = new Request("PUT", regularIndex); + regularIdxRequest.setJsonEntity("{\"aliases\": {\"" + regularAlias + "\": {}}}"); + assertOK(client().performRequest(regularIdxRequest)); + assertThatAPIWildcardResolutionWorks( + singletonList( + "this request accesses system indices: [.fleet-artifacts-7], but in a future major version, direct access to system" + + " indices will be prevented by default" + ) + ); + assertThatAPIWildcardResolutionWorks(emptyList(), "r*"); + } + + private void assertThatAPIWildcardResolutionWorks() throws Exception { + assertThatAPIWildcardResolutionWorks(emptyList(), null); + } + + private void assertThatAPIWildcardResolutionWorks(List warningsExpected) throws Exception { + assertThatAPIWildcardResolutionWorks(warningsExpected, null); + } + + private void assertThatAPIWildcardResolutionWorks(List warningsExpected, String indexPattern) throws Exception { + String path = indexPattern == null || indexPattern.isEmpty() ? "/_count" : "/" + indexPattern + "/_count"; + Request countRequest = new Request("GET", path); + if (warningsExpected.isEmpty() == false) { + countRequest.setOptions( + countRequest.getOptions().toBuilder().setWarningsHandler(warnings -> warningsExpected.equals(warnings) == false) + ); + } + assertOK(client().performRequest(countRequest)); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index d875f0a8ac5f3..2a94f627812ad 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -141,7 +141,7 @@ ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, Metadata if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) { if (replaceWildcards) { for (String authorizedIndex : authorizedIndices) { - if (IndexAbstractionResolver.isIndexVisible("*", authorizedIndex, indicesOptions, metadata, + if (IndexAbstractionResolver.isIndexVisible("*", authorizedIndex, indicesOptions, metadata, nameExpressionResolver, indicesRequest.includeDataStreams())) { resolvedIndicesBuilder.addLocal(authorizedIndex); }