Skip to content

Commit

Permalink
Security Tokens moved to a new separate index (elastic#40742)
Browse files Browse the repository at this point in the history 
This commit introduces the `.security-tokens` and `.security-tokens-7`
alias-index pair. Because index snapshotting is at the index level granularity
(ie you cannot snapshot a subset of an index) snapshoting .`security` had
the undesirable effect of storing ephemeral security tokens. The changes
herein address this issue by moving tokens "seamlessly" (without user
intervention) to another index, so that a "Security Backup" (ie snapshot of
`.security`) would not be bloated by ephemeral data.
  • Loading branch information
@albertzaharovits
albertzaharovits committed Apr 30, 2019
1 parent fe5789a commit 189c5b8
Show file tree
Hide file tree
Showing 50 changed files with 2,024 additions and 847 deletions.
13 changes: 8 additions & 5 deletions ...ore/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@
import java.util.Set;

public final class RestrictedIndicesNames {
public static final String INTERNAL_SECURITY_INDEX_6 = ".security-6";
public static final String INTERNAL_SECURITY_INDEX_7 = ".security-7";
public static final String SECURITY_INDEX_NAME = ".security";
public static final String INTERNAL_SECURITY_MAIN_INDEX_6 = ".security-6";
public static final String INTERNAL_SECURITY_MAIN_INDEX_7 = ".security-7";
public static final String SECURITY_MAIN_ALIAS = ".security";

public static final Set<String> RESTRICTED_NAMES = Collections.unmodifiableSet(
Sets.newHashSet(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX_6, INTERNAL_SECURITY_INDEX_7));
public static final String INTERNAL_SECURITY_TOKENS_INDEX_7 = ".security-tokens-7";
public static final String SECURITY_TOKENS_ALIAS = ".security-tokens";

public static final Set<String> RESTRICTED_NAMES = Collections.unmodifiableSet(Sets.newHashSet(SECURITY_MAIN_ALIAS,
INTERNAL_SECURITY_MAIN_INDEX_6, INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_SECURITY_TOKENS_INDEX_7, SECURITY_TOKENS_ALIAS));

public static final Automaton NAMES_AUTOMATON = Automatons.patterns(RESTRICTED_NAMES);

Expand Down
2 changes: 1 addition & 1 deletion ...in/resources/security-index-template.json → .../resources/security-index-template-7.json
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"index_patterns" : [ ".security-*" ],
"index_patterns" : [ ".security-7" ],
"order" : 1000,
"settings" : {
"number_of_shards" : 1,
Expand Down
96 changes: 96 additions & 0 deletions x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
{
"index_patterns" : [ ".security-tokens-7" ],
"order" : 1000,
"settings" : {
"number_of_shards" : 1,
"number_of_replicas" : 0,
"auto_expand_replicas" : "0-1",
"index.priority": 1000,
"index.format": 7
},
"mappings" : {
"_doc" : {
"_meta": {
"security-version": "${security.template.version}"
},
"dynamic" : "strict",
"properties" : {
"doc_type" : {
"type" : "keyword"
},
"creation_time" : {
"type" : "date",
"format" : "epoch_millis"
},
"refresh_token" : {
"type" : "object",
"properties" : {
"token" : {
"type" : "keyword"
},
"refreshed" : {
"type" : "boolean"
},
"refresh_time": {
"type": "date",
"format": "epoch_millis"
},
"superseded_by": {
"type": "keyword"
},
"invalidated" : {
"type" : "boolean"
},
"client" : {
"type" : "object",
"properties" : {
"type" : {
"type" : "keyword"
},
"user" : {
"type" : "keyword"
},
"realm" : {
"type" : "keyword"
}
}
}
}
},
"access_token" : {
"type" : "object",
"properties" : {
"user_token" : {
"type" : "object",
"properties" : {
"id" : {
"type" : "keyword"
},
"expiration_time" : {
"type" : "date",
"format" : "epoch_millis"
},
"version" : {
"type" : "integer"
},
"metadata" : {
"type" : "object",
"dynamic" : false
},
"authentication" : {
"type" : "binary"
}
}
},
"invalidated" : {
"type" : "boolean"
},
"realm" : {
"type" : "keyword"
}
}
}
}
}
}
}
22 changes: 11 additions & 11 deletions .../test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -601,25 +601,25 @@ public void testRemoteMonitoringCollectorRole() {

private void assertMonitoringOnRestrictedIndices(Role role) {
final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6,
RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7);
final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6,
RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7);
final MetaData metaData = new MetaData.Builder()
.put(new IndexMetaData.Builder(internalSecurityIndex)
.settings(indexSettings)
.numberOfShards(1)
.numberOfReplicas(0)
.putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
.putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build())
.build(), true)
.build();
final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
final List<String> indexMonitoringActionNamesList = Arrays.asList(IndicesStatsAction.NAME, IndicesSegmentsAction.NAME,
GetSettingsAction.NAME, IndicesShardStoresAction.NAME, UpgradeStatusAction.NAME, RecoveryAction.NAME);
for (final String indexMonitoringActionName : indexMonitoringActionNamesList) {
final Map<String, IndexAccessControl> authzMap = role.indices().authorize(indexMonitoringActionName,
Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_INDEX_NAME),
Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_MAIN_ALIAS),
metaData.getAliasAndIndexLookup(), fieldPermissionsCache);
assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(true));
assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(true));
}
}

Expand Down Expand Up @@ -713,8 +713,8 @@ public void testSuperuserRole() {
assertThat(superuserRole.cluster().check("internal:admin/foo", request), is(false));

final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6,
RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7);
final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6,
RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7);
final MetaData metaData = new MetaData.Builder()
.put(new IndexMetaData.Builder("a1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
.put(new IndexMetaData.Builder("a2").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
Expand All @@ -731,7 +731,7 @@ public void testSuperuserRole() {
.settings(indexSettings)
.numberOfShards(1)
.numberOfReplicas(0)
.putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
.putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build())
.build(), true)
.build();

Expand All @@ -753,16 +753,16 @@ public void testSuperuserRole() {
assertThat(authzMap.get("aaaaaa").isGranted(), is(true));
assertThat(authzMap.get("b").isGranted(), is(true));
authzMap = superuserRole.indices().authorize(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME),
Sets.newHashSet(RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup, fieldPermissionsCache);
assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
Sets.newHashSet(RestrictedIndicesNames.SECURITY_MAIN_ALIAS), lookup, fieldPermissionsCache);
assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(true));
assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(true));
assertTrue(superuserRole.indices().check(SearchAction.NAME));
assertFalse(superuserRole.indices().check("unknown"));

assertThat(superuserRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(true));

assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME))
.test(RestrictedIndicesNames.SECURITY_INDEX_NAME), is(true));
.test(RestrictedIndicesNames.SECURITY_MAIN_ALIAS), is(true));
assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME))
.test(internalSecurityIndex), is(true));
}
Expand Down
19 changes: 10 additions & 9 deletions x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,9 +258,9 @@
import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING;
import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING;
import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT;
import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7;

public class Security extends Plugin implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin,
DiscoveryPlugin, MapperPlugin, ExtensiblePlugin {
Expand Down Expand Up @@ -406,9 +406,10 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
components.add(auditTrailService);
this.auditTrailService.set(auditTrailService);

securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService));
securityIndex.set(SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService));

final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService);
final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(),
SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService), clusterService);
this.tokenService.set(tokenService);
components.add(tokenService);

Expand Down Expand Up @@ -948,7 +949,7 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(final Settings settings) {
public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
return templates -> {
// .security index is not managed by using templates anymore
templates.remove(SECURITY_TEMPLATE_NAME);
templates.remove(SECURITY_MAIN_TEMPLATE_7);
templates.remove("security_audit_log");
return templates;
};
Expand Down Expand Up @@ -1015,9 +1016,9 @@ static final class ValidateUpgradedSecurityIndex implements BiConsumer<Discovery
@Override
public void accept(DiscoveryNode node, ClusterState state) {
if (state.getNodes().getMinNodeVersion().before(Version.V_7_0_0)) {
IndexMetaData indexMetaData = state.getMetaData().getIndices().get(SECURITY_INDEX_NAME);
if (indexMetaData != null && INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()) < INTERNAL_INDEX_FORMAT) {
throw new IllegalStateException("Security index is not on the current version [" + INTERNAL_INDEX_FORMAT + "] - " +
IndexMetaData indexMetaData = state.getMetaData().getIndices().get(SECURITY_MAIN_ALIAS);
if (indexMetaData != null && INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()) < INTERNAL_MAIN_INDEX_FORMAT) {
throw new IllegalStateException("Security index is not on the current version [" + INTERNAL_MAIN_INDEX_FORMAT + "] - " +
"The Upgrade API must be run for 7.x nodes to join the cluster");
}
}
Expand Down
12 changes: 6 additions & 6 deletions ...k/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@
import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING;
import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;

public class ApiKeyService {

Expand Down Expand Up @@ -207,7 +207,7 @@ private void checkDuplicateApiKeyNameAndCreateApiKey(Authentication authenticati
.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time")));
boolQuery.filter(expiredQuery);

final SearchRequest searchRequest = client.prepareSearch(SECURITY_INDEX_NAME)
final SearchRequest searchRequest = client.prepareSearch(SECURITY_MAIN_ALIAS)
.setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
.setQuery(boolQuery)
.setVersion(false)
Expand Down Expand Up @@ -286,7 +286,7 @@ private void createApiKeyAndIndexIt(Authentication authentication, CreateApiKeyR
.endObject()
.endObject();
final IndexRequest indexRequest =
client.prepareIndex(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME)
client.prepareIndex(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME)
.setSource(builder)
.setRefreshPolicy(request.getRefreshPolicy())
.request();
Expand Down Expand Up @@ -319,7 +319,7 @@ void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<Authentic

if (credentials != null) {
final GetRequest getRequest = client
.prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, credentials.getId())
.prepareGet(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, credentials.getId())
.setFetchSource(true)
.request();
executeAsyncWithOrigin(ctx, SECURITY_ORIGIN, getRequest, ActionListener.<GetResponse>wrap(response -> {
Expand Down Expand Up @@ -727,7 +727,7 @@ private void findApiKeys(final BoolQueryBuilder boolQuery, boolean filterOutInva
expiredQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time")));
boolQuery.filter(expiredQuery);
}
final SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME)
final SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS)
.setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
.setQuery(boolQuery)
.setVersion(false)
Expand Down Expand Up @@ -801,7 +801,7 @@ private void indexInvalidation(Collection<String> apiKeyIds, ActionListener<Inva
BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (String apiKeyId : apiKeyIds) {
UpdateRequest request = client
.prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, apiKeyId)
.prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, apiKeyId)
.setDoc(Collections.singletonMap("api_key_invalidated", true))
.request();
bulkRequestBuilder.add(request);
Expand Down
4 changes: 2 additions & 2 deletions .../security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredApiKeysRemover.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.elasticsearch.index.reindex.ScrollableHitSource;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.threadpool.ThreadPool.Names;
import org.elasticsearch.xpack.security.support.SecurityIndexManager;
import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;

import java.time.Duration;
import java.time.Instant;
Expand Down Expand Up @@ -51,7 +51,7 @@ public final class ExpiredApiKeysRemover extends AbstractRunnable {

@Override
public void doRun() {
DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(SecurityIndexManager.SECURITY_INDEX_NAME);
DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(RestrictedIndicesNames.SECURITY_MAIN_ALIAS);
if (timeout != TimeValue.MINUS_ONE) {
expiredDbq.setTimeout(timeout);
expiredDbq.getSearchRequest().source().timeout(timeout);
Expand Down
Loading

0 comments on commit 189c5b8

Please sign in to comment.