Security Tokens moved to a new separate index

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java

@@ -14,12 +14,15 @@
 import java.util.Set;
 
 public final class RestrictedIndicesNames {
-    public static final String INTERNAL_SECURITY_INDEX_6 = ".security-6";
-    public static final String INTERNAL_SECURITY_INDEX_7 = ".security-7";
-    public static final String SECURITY_INDEX_NAME = ".security";
+    public static final String INTERNAL_SECURITY_MAIN_INDEX_6 = ".security-6";
+    public static final String INTERNAL_SECURITY_MAIN_INDEX_7 = ".security-7";
+    public static final String SECURITY_MAIN_ALIAS = ".security";
 
-    public static final Set<String> RESTRICTED_NAMES = Collections.unmodifiableSet(
-        Sets.newHashSet(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX_6, INTERNAL_SECURITY_INDEX_7));
+    public static final String INTERNAL_SECURITY_TOKENS_INDEX_7 = ".security-tokens-7";
+    public static final String SECURITY_TOKENS_ALIAS = ".security-tokens";
+
+    public static final Set<String> RESTRICTED_NAMES = Collections.unmodifiableSet(Sets.newHashSet(SECURITY_MAIN_ALIAS,
+            INTERNAL_SECURITY_MAIN_INDEX_6, INTERNAL_SECURITY_MAIN_INDEX_7, INTERNAL_SECURITY_TOKENS_INDEX_7, SECURITY_TOKENS_ALIAS));
 
     public static final Automaton NAMES_AUTOMATON = Automatons.patterns(RESTRICTED_NAMES);
 

x-pack/plugin/core/src/main/resources/security-index-template.json → x-pack/plugin/core/src/main/resources/security-index-template-7.json

@@ -1,5 +1,5 @@
 {
-  "index_patterns" : [ ".security-*" ],
+  "index_patterns" : [ ".security-7" ],
   "order" : 1000,
   "settings" : {
     "number_of_shards" : 1,

x-pack/plugin/core/src/main/resources/security-tokens-index-template-7.json

@@ -0,0 +1,96 @@
+{
+  "index_patterns" : [ ".security-tokens-7" ],
+  "order" : 1000,
+  "settings" : {
+    "number_of_shards" : 1,
+    "number_of_replicas" : 0,
+    "auto_expand_replicas" : "0-1",
+    "index.priority": 1000,
+    "index.format": 7
+  },
+  "mappings" : {
+    "_doc" : {
+      "_meta": {
+        "security-version": "${security.template.version}"
+      },
+      "dynamic" : "strict",
+      "properties" : {
+        "doc_type" : {
+          "type" : "keyword"
+        },
+        "creation_time" : {
+          "type" : "date",
+          "format" : "epoch_millis"
+        },
+        "refresh_token" : {
+          "type" : "object",
+          "properties" : {
+            "token" : {
+              "type" : "keyword"
+            },
+            "refreshed" : {
+              "type" : "boolean"
+            },
+            "refresh_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "superseded_by": {
+              "type": "keyword"
+            },
+            "invalidated" : {
+              "type" : "boolean"
+            },
+            "client" : {
+              "type" : "object",
+              "properties" : {
+                "type" : {
+                  "type" : "keyword"
+                },
+                "user" : {
+                  "type" : "keyword"
+                },
+                "realm" : {
+                  "type" : "keyword"
+                }
+              }
+            }
+          }
+        },
+        "access_token" : {
+          "type" : "object",
+          "properties" : {
+            "user_token" : {
+              "type" : "object",
+              "properties" : {
+                "id" : {
+                  "type" : "keyword"
+                },
+                "expiration_time" : {
+                  "type" : "date",
+                  "format" : "epoch_millis"
+                },
+                "version" : {
+                  "type" : "integer"
+                },
+                "metadata" : {
+                  "type" : "object",
+                  "dynamic" : false
+                },
+                "authentication" : {
+                  "type" : "binary"
+                }
+              }
+            },
+            "invalidated" : {
+              "type" : "boolean"
+            },
+            "realm" : {
+              "type" : "keyword"
+            }
+          }
+        }
+      }
+    }
+  }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java

@@ -601,25 +601,25 @@ public void testRemoteMonitoringCollectorRole() {
 
     private void assertMonitoringOnRestrictedIndices(Role role) {
         final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
-        final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6,
-            RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7);
+        final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6,
+            RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7);
         final MetaData metaData = new MetaData.Builder()
                 .put(new IndexMetaData.Builder(internalSecurityIndex)
                         .settings(indexSettings)
                         .numberOfShards(1)
                         .numberOfReplicas(0)
-                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
+                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build())
                         .build(), true)
                 .build();
         final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
         final List<String> indexMonitoringActionNamesList = Arrays.asList(IndicesStatsAction.NAME, IndicesSegmentsAction.NAME,
                 GetSettingsAction.NAME, IndicesShardStoresAction.NAME, UpgradeStatusAction.NAME, RecoveryAction.NAME);
         for (final String indexMonitoringActionName : indexMonitoringActionNamesList) {
             final Map<String, IndexAccessControl> authzMap = role.indices().authorize(indexMonitoringActionName,
-                Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_INDEX_NAME),
+                Sets.newHashSet(internalSecurityIndex, RestrictedIndicesNames.SECURITY_MAIN_ALIAS),
                 metaData.getAliasAndIndexLookup(), fieldPermissionsCache);
             assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(true));
-            assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
+            assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(true));
         }
     }
 
@@ -713,8 +713,8 @@ public void testSuperuserRole() {
         assertThat(superuserRole.cluster().check("internal:admin/foo", request), is(false));
 
         final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
-        final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_6,
-            RestrictedIndicesNames.INTERNAL_SECURITY_INDEX_7);
+        final String internalSecurityIndex = randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6,
+            RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7);
         final MetaData metaData = new MetaData.Builder()
                 .put(new IndexMetaData.Builder("a1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
                 .put(new IndexMetaData.Builder("a2").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
@@ -731,7 +731,7 @@ public void testSuperuserRole() {
                         .settings(indexSettings)
                         .numberOfShards(1)
                         .numberOfReplicas(0)
-                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
+                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).build())
                         .build(), true)
                 .build();
 
@@ -753,16 +753,16 @@ public void testSuperuserRole() {
         assertThat(authzMap.get("aaaaaa").isGranted(), is(true));
         assertThat(authzMap.get("b").isGranted(), is(true));
         authzMap = superuserRole.indices().authorize(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME),
-                Sets.newHashSet(RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup, fieldPermissionsCache);
-        assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
+                Sets.newHashSet(RestrictedIndicesNames.SECURITY_MAIN_ALIAS), lookup, fieldPermissionsCache);
+        assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_MAIN_ALIAS).isGranted(), is(true));
         assertThat(authzMap.get(internalSecurityIndex).isGranted(), is(true));
         assertTrue(superuserRole.indices().check(SearchAction.NAME));
         assertFalse(superuserRole.indices().check("unknown"));
 
         assertThat(superuserRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(true));
 
         assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME))
-                .test(RestrictedIndicesNames.SECURITY_INDEX_NAME), is(true));
+                .test(RestrictedIndicesNames.SECURITY_MAIN_ALIAS), is(true));
         assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME))
                 .test(internalSecurityIndex), is(true));
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -258,9 +258,9 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
-import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT;
-import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
-import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME;
+import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT;
+import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
+import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7;
 
 public class Security extends Plugin implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin,
         DiscoveryPlugin, MapperPlugin, ExtensiblePlugin {
@@ -406,9 +406,10 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         components.add(auditTrailService);
         this.auditTrailService.set(auditTrailService);
 
-        securityIndex.set(SecurityIndexManager.buildSecurityIndexManager(client, clusterService));
+        securityIndex.set(SecurityIndexManager.buildSecurityMainIndexManager(client, clusterService));
 
-        final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService);
+        final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(),
+                SecurityIndexManager.buildSecurityTokensIndexManager(client, clusterService), clusterService);
         this.tokenService.set(tokenService);
         components.add(tokenService);
 
@@ -965,7 +966,7 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(final Settings settings) {
     public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
         return templates -> {
             // .security index is not managed by using templates anymore
-            templates.remove(SECURITY_TEMPLATE_NAME);
+            templates.remove(SECURITY_MAIN_TEMPLATE_7);
             templates.remove("security_audit_log");
             return templates;
         };
@@ -1031,9 +1032,9 @@ static final class ValidateUpgradedSecurityIndex implements BiConsumer<Discovery
         @Override
         public void accept(DiscoveryNode node, ClusterState state) {
             if (state.getNodes().getMinNodeVersion().before(Version.V_7_0_0)) {
-                IndexMetaData indexMetaData = state.getMetaData().getIndices().get(SECURITY_INDEX_NAME);
-                if (indexMetaData != null && INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()) < INTERNAL_INDEX_FORMAT) {
-                    throw new IllegalStateException("Security index is not on the current version [" + INTERNAL_INDEX_FORMAT + "] - " +
+                IndexMetaData indexMetaData = state.getMetaData().getIndices().get(SECURITY_MAIN_ALIAS);
+                if (indexMetaData != null && INDEX_FORMAT_SETTING.get(indexMetaData.getSettings()) < INTERNAL_MAIN_INDEX_FORMAT) {
+                    throw new IllegalStateException("Security index is not on the current version [" + INTERNAL_MAIN_INDEX_FORMAT + "] - " +
                         "The Upgrade API must be run for 7.x nodes to join the cluster");
                 }
             }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -98,7 +98,7 @@
 import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING;
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
-import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
+import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 
 public class ApiKeyService {
 
@@ -207,7 +207,7 @@ private void checkDuplicateApiKeyNameAndCreateApiKey(Authentication authenticati
                 .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time")));
         boolQuery.filter(expiredQuery);
 
-        final SearchRequest searchRequest = client.prepareSearch(SECURITY_INDEX_NAME)
+        final SearchRequest searchRequest = client.prepareSearch(SECURITY_MAIN_ALIAS)
             .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
             .setQuery(boolQuery)
             .setVersion(false)
@@ -280,7 +280,7 @@ private void createApiKeyAndIndexIt(Authentication authentication, CreateApiKeyR
                 .endObject()
                 .endObject();
             final IndexRequest indexRequest =
-                client.prepareIndex(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME)
+                client.prepareIndex(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME)
                     .setSource(builder)
                     .setRefreshPolicy(request.getRefreshPolicy())
                     .request();
@@ -313,7 +313,7 @@ void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<Authentic
 
             if (credentials != null) {
                 final GetRequest getRequest = client
-                        .prepareGet(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, credentials.getId())
+                        .prepareGet(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, credentials.getId())
                         .setFetchSource(true)
                         .request();
                 executeAsyncWithOrigin(ctx, SECURITY_ORIGIN, getRequest, ActionListener.<GetResponse>wrap(response -> {
@@ -721,7 +721,7 @@ private void findApiKeys(final BoolQueryBuilder boolQuery, boolean filterOutInva
             expiredQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time")));
             boolQuery.filter(expiredQuery);
         }
-        final SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME)
+        final SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS)
             .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
             .setQuery(boolQuery)
             .setVersion(false)
@@ -795,7 +795,7 @@ private void indexInvalidation(Collection<String> apiKeyIds, ActionListener<Inva
             BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
             for (String apiKeyId : apiKeyIds) {
                 UpdateRequest request = client
-                        .prepareUpdate(SECURITY_INDEX_NAME, SINGLE_MAPPING_NAME, apiKeyId)
+                        .prepareUpdate(SECURITY_MAIN_ALIAS, SINGLE_MAPPING_NAME, apiKeyId)
                         .setDoc(Collections.singletonMap("api_key_invalidated", true))
                         .request();
                 bulkRequestBuilder.add(request);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredApiKeysRemover.java

@@ -22,7 +22,7 @@
 import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
-import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -51,7 +51,7 @@ public final class ExpiredApiKeysRemover extends AbstractRunnable {
 
     @Override
     public void doRun() {
-        DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(SecurityIndexManager.SECURITY_INDEX_NAME);
+        DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(RestrictedIndicesNames.SECURITY_MAIN_ALIAS);
         if (timeout != TimeValue.MINUS_ONE) {
             expiredDbq.setTimeout(timeout);
             expiredDbq.getSearchRequest().source().timeout(timeout);