APM server monitoring

docs/reference/commands/setup-passwords.asciidoc

@@ -4,7 +4,7 @@
 == elasticsearch-setup-passwords
 
 The `elasticsearch-setup-passwords` command sets the passwords for the built-in
-`elastic`, `kibana`, `logstash_system`, and `beats_system` users.
+`elastic`, `kibana`, `logstash_system`, `beats_system`, and `apm_system` users.
 
 [float]
 === Synopsis

docs/reference/monitoring/exporters.asciidoc

@@ -105,12 +105,12 @@ route monitoring data:
 
 [options="header"]
 |=======================
-| Template               | Purpose
-| `.monitoring-alerts`   | All cluster alerts for monitoring data.
-| `.monitoring-beats`    | All Beats monitoring data.
-| `.monitoring-es`       | All {es} monitoring data.
-| `.monitoring-kibana`   | All {kib} monitoring data.
-| `.monitoring-logstash` | All Logstash monitoring data.
+| Template                    | Purpose
+| `.monitoring-alerts`        | All cluster alerts for monitoring data.
+| `.monitoring-beats`         | All Beats monitoring data.
+| `.monitoring-es`            | All {es} monitoring data.
+| `.monitoring-kibana`        | All {kib} monitoring data.
+| `.monitoring-logstash`      | All Logstash monitoring data.
 |=======================
 
 The templates are ordinary {es} templates that control the default settings and

x-pack/docs/en/security/configuring-es.asciidoc

@@ -55,8 +55,8 @@ help you get up and running. The +elasticsearch-setup-passwords+ command is the
 simplest method to set the built-in users' passwords for the first time.
 
 For example, you can run the command in an "interactive" mode, which prompts you
-to enter new passwords for the `elastic`, `kibana`, `beats_system`, and
-`logstash_system` users:
+to enter new passwords for the `elastic`, `kibana`, `beats_system`,
+`logstash_system`, and `apm_system` users:
 
 [source,shell]
 --------------------------------------------------

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/esnative/ClientReservedRealm.java

@@ -19,6 +19,7 @@ public static boolean isReserved(String username, Settings settings) {
             case UsernamesField.KIBANA_NAME:
             case UsernamesField.LOGSTASH_NAME:
             case UsernamesField.BEATS_NAME:
+            case UsernamesField.APM_NAME:
                 return XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(settings);
             default:
                 return AnonymousUser.isAnonymousUsername(username, settings);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java

@@ -112,6 +112,8 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                     null, MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put(UsernamesField.BEATS_ROLE, new RoleDescriptor(UsernamesField.BEATS_ROLE,
                         new String[] { "monitor", MonitoringBulkAction.NAME}, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
+                .put(UsernamesField.APM_ROLE, new RoleDescriptor(UsernamesField.APM_ROLE,
+                        new String[] { "monitor", MonitoringBulkAction.NAME}, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("machine_learning_user", new RoleDescriptor("machine_learning_user", new String[] { "monitor_ml" },
                         new RoleDescriptor.IndicesPrivileges[] { RoleDescriptor.IndicesPrivileges.builder().indices(".ml-anomalies*",
                                 ".ml-notifications").privileges("view_index_metadata", "read").build() },

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.user;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.protocol.xpack.security.User;
+import org.elasticsearch.xpack.core.security.support.MetadataUtils;
+
+/**
+ * Built in user for APM server internals. Currently used for APM server monitoring.
+ */
+public class APMSystemUser extends User {
+
+    public static final String NAME = UsernamesField.APM_NAME;
+    public static final String ROLE_NAME = UsernamesField.APM_ROLE;
+    public static final Version DEFINED_SINCE = Version.V_6_5_0;
+    public static final BuiltinUserInfo USER_INFO = new BuiltinUserInfo(NAME, ROLE_NAME, DEFINED_SINCE);
+
+    public APMSystemUser(boolean enabled) {
+        super(NAME, new String[]{ ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/UsernamesField.java

@@ -20,6 +20,8 @@ public final class UsernamesField {
     public static final String LOGSTASH_ROLE = "logstash_system";
     public static final String BEATS_NAME = "beats_system";
     public static final String BEATS_ROLE = "beats_system";
+    public static final String APM_NAME = "apm_system";
+    public static final String APM_ROLE = "apm_system";
 
     private UsernamesField() {}
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java

@@ -94,6 +94,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
@@ -147,6 +148,7 @@ public void testIsReserved() {
         assertThat(ReservedRolesStore.isReserved(XPackUser.ROLE_NAME), is(true));
         assertThat(ReservedRolesStore.isReserved(LogstashSystemUser.ROLE_NAME), is(true));
         assertThat(ReservedRolesStore.isReserved(BeatsSystemUser.ROLE_NAME), is(true));
+        assertThat(ReservedRolesStore.isReserved(APMSystemUser.ROLE_NAME), is(true));
     }
 
     public void testIngestAdminRole() {
@@ -628,6 +630,30 @@ public void testBeatsSystemRole() {
                 is(false));
     }
 
+    public void testAPMSystemRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
+        RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor(APMSystemUser.ROLE_NAME);
+        assertNotNull(roleDescriptor);
+        assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
+
+        Role APMSystemRole = Role.builder(roleDescriptor, null).build();
+        assertThat(APMSystemRole.cluster().check(ClusterHealthAction.NAME, request), is(true));
+        assertThat(APMSystemRole.cluster().check(ClusterStateAction.NAME, request), is(true));
+        assertThat(APMSystemRole.cluster().check(ClusterStatsAction.NAME, request), is(true));
+        assertThat(APMSystemRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(APMSystemRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(APMSystemRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(APMSystemRole.cluster().check(MonitoringBulkAction.NAME, request), is(true));
+
+        assertThat(APMSystemRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
+
+        assertThat(APMSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test("foo"), is(false));
+        assertThat(APMSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(".reporting"), is(false));
+        assertThat(APMSystemRole.indices().allowedIndicesMatcher("indices:foo").test(randomAlphaOfLengthBetween(8, 24)),
+                is(false));
+    }
+
     public void testMachineLearningAdminRole() {
         final TransportRequest request = mock(TransportRequest.class);
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
@@ -149,6 +150,8 @@ private User getUser(String username, ReservedUserInfo userInfo) {
                 return new LogstashSystemUser(userInfo.enabled);
             case BeatsSystemUser.NAME:
                 return new BeatsSystemUser(userInfo.enabled);
+            case APMSystemUser.NAME:
+                return new APMSystemUser(userInfo.enabled);
             default:
                 if (anonymousEnabled && anonymousUser.principal().equals(username)) {
                     return anonymousUser;
@@ -177,6 +180,9 @@ public void users(ActionListener<Collection<User>> listener) {
                 userInfo = reservedUserInfos.get(BeatsSystemUser.NAME);
                 users.add(new BeatsSystemUser(userInfo == null || userInfo.enabled));
 
+                userInfo = reservedUserInfos.get(APMSystemUser.NAME);
+                users.add(new APMSystemUser(userInfo == null || userInfo.enabled));
+
                 if (anonymousEnabled) {
                     users.add(anonymousUser);
                 }
@@ -230,6 +236,8 @@ private Version getDefinedVersion(String username) {
                 return LogstashSystemUser.DEFINED_SINCE;
             case BeatsSystemUser.NAME:
                 return BeatsSystemUser.DEFINED_SINCE;
+            case APMSystemUser.NAME:
+                return APMSystemUser.DEFINED_SINCE;
             default:
                 return Version.V_5_0_0;
         }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.support.Validation;
+import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
@@ -63,7 +64,8 @@
 public class SetupPasswordTool extends LoggingAwareMultiCommand {
 
     private static final char[] CHARS = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789").toCharArray();
-    public static final List<String> USERS = asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
+    public static final List<String> USERS = asList(ElasticUser.NAME, APMSystemUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
+        BeatsSystemUser.NAME);
 
     private final BiFunction<Environment, Settings, CommandLineHttpClient> clientFunction;
     private final CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction;

x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
+import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
@@ -88,7 +89,7 @@ public void setupReservedPasswords(RestClient restClient) throws IOException {
         RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
         optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, reservedPassword));
         RequestOptions options = optionsBuilder.build();
-        for (String username : Arrays.asList(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
+        for (String username : Arrays.asList(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME, APMSystemUser.NAME)) {
             Request request = new Request("PUT", "/_xpack/security/user/" + username + "/_password");
             request.setJsonEntity("{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}");
             request.setOptions(options);

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
@@ -81,7 +82,8 @@ void doExecute(Action<Response> action, Request request, ActionListener<Response
     public void testPasswordUpsertWhenSetEnabledOnReservedUser() throws Exception {
         final NativeUsersStore nativeUsersStore = startNativeUsersStore();
 
-        final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
+        final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
+            BeatsSystemUser.NAME, APMSystemUser.NAME);
 
         final PlainActionFuture<Void> future = new PlainActionFuture<>();
         nativeUsersStore.setEnabled(user, true, WriteRequest.RefreshPolicy.IMMEDIATE, future);
@@ -99,7 +101,8 @@ public void testPasswordUpsertWhenSetEnabledOnReservedUser() throws Exception {
     public void testBlankPasswordInIndexImpliesDefaultPassword() throws Exception {
         final NativeUsersStore nativeUsersStore = startNativeUsersStore();
 
-        final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
+        final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
+            BeatsSystemUser.NAME, APMSystemUser.NAME);
         final Map<String, Object> values = new HashMap<>();
         values.put(ENABLED_FIELD, Boolean.TRUE);
         values.put(PASSWORD_FIELD, BLANK_PASSWORD);

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java

@@ -13,13 +13,15 @@
 import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
+import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
 import org.junit.BeforeClass;
 
 import java.util.Arrays;
+import java.util.List;
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
@@ -49,7 +51,9 @@ public Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testAuthenticate() {
-        for (String username : Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
+        final List<String> usernames = Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
+            BeatsSystemUser.NAME, APMSystemUser.NAME);
+        for (String username : usernames) {
             ClusterHealthResponse response = client()
                     .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(username, getReservedPassword())))
                     .admin()
@@ -67,7 +71,9 @@ public void testAuthenticate() {
      */
     public void testAuthenticateAfterEnablingUser() {
         final SecurityClient c = securityClient();
-        for (String username : Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
+        final List<String> usernames = Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
+            BeatsSystemUser.NAME, APMSystemUser.NAME);
+        for (String username : usernames) {
             c.prepareSetEnabled(username, true).get();
             ClusterHealthResponse response = client()
                     .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(username, getReservedPassword())))
@@ -81,7 +87,8 @@ public void testAuthenticateAfterEnablingUser() {
     }
 
     public void testChangingPassword() {
-        String username = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
+        String username = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
+            BeatsSystemUser.NAME, APMSystemUser.NAME);
         final char[] newPassword = "supersecretvalue".toCharArray();
 
         if (randomBoolean()) {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
@@ -262,7 +263,8 @@ public void testGetUsers() {
         PlainActionFuture<Collection<User>> userFuture = new PlainActionFuture<>();
         reservedRealm.users(userFuture);
         assertThat(userFuture.actionGet(),
-            containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true), new BeatsSystemUser(true)));
+            containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true),
+                new BeatsSystemUser(true), new APMSystemUser((true))));
     }
 
     public void testGetUsersDisabled() {
@@ -394,7 +396,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexExists
             new AnonymousUser(Settings.EMPTY), securityIndex, threadPool);
         PlainActionFuture<AuthenticationResult> listener = new PlainActionFuture<>();
 
-        final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
+        final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME, APMSystemUser.NAME);
         doAnswer((i) -> {
             ActionListener callback = (ActionListener) i.getArguments()[1];
             callback.onResponse(null);
@@ -416,14 +418,15 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexDoesNo
             new AnonymousUser(Settings.EMPTY), securityIndex, threadPool);
         PlainActionFuture<AuthenticationResult> listener = new PlainActionFuture<>();
 
-        final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
+        final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME, APMSystemUser.NAME);
         reservedRealm.doAuthenticate(new UsernamePasswordToken(principal, mockSecureSettings.getString("bootstrap.password")), listener);
         final AuthenticationResult result = listener.get();
         assertThat(result.getStatus(), is(AuthenticationResult.Status.TERMINATE));
     }
 
     private User randomReservedUser(boolean enabled) {
-        return randomFrom(new ElasticUser(enabled), new KibanaUser(enabled), new LogstashSystemUser(enabled), new BeatsSystemUser(enabled));
+        return randomFrom(new ElasticUser(enabled), new KibanaUser(enabled), new LogstashSystemUser(enabled),
+            new BeatsSystemUser(enabled), new APMSystemUser(enabled));
     }
 
     /*
@@ -457,6 +460,11 @@ private void verifyVersionPredicate(String principal, Predicate<Version> version
                 assertThat(versionPredicate.test(Version.V_6_2_3), is(false));
                 assertThat(versionPredicate.test(Version.V_6_3_0), is(true));
                 break;
+            case APMSystemUser.NAME:
+                assertThat(versionPredicate.test(Version.V_5_6_9), is(false));
+                assertThat(versionPredicate.test(Version.V_6_4_0), is(false));
+                assertThat(versionPredicate.test(Version.V_6_5_0), is(true));
+                break;
             default:
                 assertThat(versionPredicate.test(Version.V_5_0_0), is(true));
                 assertThat(versionPredicate.test(Version.V_5_1_1), is(true));