Skip to content

Commit

Permalink
APM server monitoring (#32515)
Browse files Browse the repository at this point in the history
* Adding new MonitoredSystem for APM server

* Teaching Monitoring template utils about APM server monitoring indices

* Documenting new monitoring index for APM server

* Adding monitoring index template for APM server

* Copy pasta typo

* Removing metrics.libbeat.config section from mapping

* Adding built-in user and role for APM server user

* Actually define the role :)

* Adding missing import

* Removing index template and system ID for apm server

* Shortening line lengths

* Updating expected number of built-in users in integration test

* Removing "system" from role and user names

* Rearranging users to make tests pass
  • Loading branch information
ycombinator authored Aug 27, 2018
1 parent f1f6d4e commit 1779d33
Show file tree
Hide file tree
Showing 15 changed files with 106 additions and 21 deletions.
2 changes: 1 addition & 1 deletion docs/reference/commands/setup-passwords.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
== elasticsearch-setup-passwords

The `elasticsearch-setup-passwords` command sets the passwords for the built-in
`elastic`, `kibana`, `logstash_system`, and `beats_system` users.
`elastic`, `kibana`, `logstash_system`, `beats_system`, and `apm_system` users.

[float]
=== Synopsis
Expand Down
12 changes: 6 additions & 6 deletions docs/reference/monitoring/exporters.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,12 @@ route monitoring data:

[options="header"]
|=======================
| Template | Purpose
| `.monitoring-alerts` | All cluster alerts for monitoring data.
| `.monitoring-beats` | All Beats monitoring data.
| `.monitoring-es` | All {es} monitoring data.
| `.monitoring-kibana` | All {kib} monitoring data.
| `.monitoring-logstash` | All Logstash monitoring data.
| Template | Purpose
| `.monitoring-alerts` | All cluster alerts for monitoring data.
| `.monitoring-beats` | All Beats monitoring data.
| `.monitoring-es` | All {es} monitoring data.
| `.monitoring-kibana` | All {kib} monitoring data.
| `.monitoring-logstash` | All Logstash monitoring data.
|=======================

The templates are ordinary {es} templates that control the default settings and
Expand Down
4 changes: 2 additions & 2 deletions x-pack/docs/en/security/configuring-es.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ help you get up and running. The +elasticsearch-setup-passwords+ command is the
simplest method to set the built-in users' passwords for the first time.

For example, you can run the command in an "interactive" mode, which prompts you
to enter new passwords for the `elastic`, `kibana`, `beats_system`, and
`logstash_system` users:
to enter new passwords for the `elastic`, `kibana`, `beats_system`,
`logstash_system`, and `apm_system` users:

[source,shell]
--------------------------------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public static boolean isReserved(String username, Settings settings) {
case UsernamesField.KIBANA_NAME:
case UsernamesField.LOGSTASH_NAME:
case UsernamesField.BEATS_NAME:
case UsernamesField.APM_NAME:
return XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(settings);
default:
return AnonymousUser.isAnonymousUsername(username, settings);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
null, MetadataUtils.DEFAULT_RESERVED_METADATA))
.put(UsernamesField.BEATS_ROLE, new RoleDescriptor(UsernamesField.BEATS_ROLE,
new String[] { "monitor", MonitoringBulkAction.NAME}, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
.put(UsernamesField.APM_ROLE, new RoleDescriptor(UsernamesField.APM_ROLE,
new String[] { "monitor", MonitoringBulkAction.NAME}, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
.put("machine_learning_user", new RoleDescriptor("machine_learning_user", new String[] { "monitor_ml" },
new RoleDescriptor.IndicesPrivileges[] { RoleDescriptor.IndicesPrivileges.builder().indices(".ml-anomalies*",
".ml-notifications").privileges("view_index_metadata", "read").build() },
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.security.user;

import org.elasticsearch.Version;
import org.elasticsearch.protocol.xpack.security.User;
import org.elasticsearch.xpack.core.security.support.MetadataUtils;

/**
* Built in user for APM server internals. Currently used for APM server monitoring.
*/
public class APMSystemUser extends User {

public static final String NAME = UsernamesField.APM_NAME;
public static final String ROLE_NAME = UsernamesField.APM_ROLE;
public static final Version DEFINED_SINCE = Version.V_6_5_0;
public static final BuiltinUserInfo USER_INFO = new BuiltinUserInfo(NAME, ROLE_NAME, DEFINED_SINCE);

public APMSystemUser(boolean enabled) {
super(NAME, new String[]{ ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ public final class UsernamesField {
public static final String LOGSTASH_ROLE = "logstash_system";
public static final String BEATS_NAME = "beats_system";
public static final String BEATS_ROLE = "beats_system";
public static final String APM_NAME = "apm_system";
public static final String APM_ROLE = "apm_system";

private UsernamesField() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
import org.elasticsearch.xpack.core.security.authz.permission.Role;
import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
import org.elasticsearch.xpack.core.security.user.APMSystemUser;
import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
import org.elasticsearch.xpack.core.security.user.SystemUser;
Expand Down Expand Up @@ -147,6 +148,7 @@ public void testIsReserved() {
assertThat(ReservedRolesStore.isReserved(XPackUser.ROLE_NAME), is(true));
assertThat(ReservedRolesStore.isReserved(LogstashSystemUser.ROLE_NAME), is(true));
assertThat(ReservedRolesStore.isReserved(BeatsSystemUser.ROLE_NAME), is(true));
assertThat(ReservedRolesStore.isReserved(APMSystemUser.ROLE_NAME), is(true));
}

public void testIngestAdminRole() {
Expand Down Expand Up @@ -628,6 +630,30 @@ public void testBeatsSystemRole() {
is(false));
}

public void testAPMSystemRole() {
final TransportRequest request = mock(TransportRequest.class);

RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor(APMSystemUser.ROLE_NAME);
assertNotNull(roleDescriptor);
assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));

Role APMSystemRole = Role.builder(roleDescriptor, null).build();
assertThat(APMSystemRole.cluster().check(ClusterHealthAction.NAME, request), is(true));
assertThat(APMSystemRole.cluster().check(ClusterStateAction.NAME, request), is(true));
assertThat(APMSystemRole.cluster().check(ClusterStatsAction.NAME, request), is(true));
assertThat(APMSystemRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
assertThat(APMSystemRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
assertThat(APMSystemRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
assertThat(APMSystemRole.cluster().check(MonitoringBulkAction.NAME, request), is(true));

assertThat(APMSystemRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));

assertThat(APMSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test("foo"), is(false));
assertThat(APMSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(".reporting"), is(false));
assertThat(APMSystemRole.indices().allowedIndicesMatcher("indices:foo").test(randomAlphaOfLengthBetween(8, 24)),
is(false));
}

public void testMachineLearningAdminRole() {
final TransportRequest request = mock(TransportRequest.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.elasticsearch.xpack.core.security.authc.support.Hasher;
import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.core.security.support.Exceptions;
import org.elasticsearch.xpack.core.security.user.APMSystemUser;
import org.elasticsearch.xpack.core.security.user.AnonymousUser;
import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.core.security.user.ElasticUser;
Expand Down Expand Up @@ -149,6 +150,8 @@ private User getUser(String username, ReservedUserInfo userInfo) {
return new LogstashSystemUser(userInfo.enabled);
case BeatsSystemUser.NAME:
return new BeatsSystemUser(userInfo.enabled);
case APMSystemUser.NAME:
return new APMSystemUser(userInfo.enabled);
default:
if (anonymousEnabled && anonymousUser.principal().equals(username)) {
return anonymousUser;
Expand Down Expand Up @@ -177,6 +180,9 @@ public void users(ActionListener<Collection<User>> listener) {
userInfo = reservedUserInfos.get(BeatsSystemUser.NAME);
users.add(new BeatsSystemUser(userInfo == null || userInfo.enabled));

userInfo = reservedUserInfos.get(APMSystemUser.NAME);
users.add(new APMSystemUser(userInfo == null || userInfo.enabled));

if (anonymousEnabled) {
users.add(anonymousUser);
}
Expand Down Expand Up @@ -228,6 +234,8 @@ private Version getDefinedVersion(String username) {
switch (username) {
case BeatsSystemUser.NAME:
return BeatsSystemUser.DEFINED_SINCE;
case APMSystemUser.NAME:
return APMSystemUser.DEFINED_SINCE;
default:
return Version.V_6_0_0;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.env.Environment;
import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.security.support.Validation;
import org.elasticsearch.xpack.core.security.user.APMSystemUser;
import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.core.security.user.ElasticUser;
import org.elasticsearch.xpack.core.security.user.KibanaUser;
Expand Down Expand Up @@ -63,7 +64,8 @@
public class SetupPasswordTool extends LoggingAwareMultiCommand {

private static final char[] CHARS = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789").toCharArray();
public static final List<String> USERS = asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
public static final List<String> USERS = asList(ElasticUser.NAME, APMSystemUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
BeatsSystemUser.NAME);

private final BiFunction<Environment, Settings, CommandLineHttpClient> clientFunction;
private final CheckedFunction<Environment, KeyStoreWrapper, Exception> keyStoreFunction;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.core.security.client.SecurityClient;
import org.elasticsearch.xpack.core.security.user.APMSystemUser;
import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.core.security.user.ElasticUser;
import org.elasticsearch.xpack.core.security.user.KibanaUser;
Expand Down Expand Up @@ -88,7 +89,7 @@ public void setupReservedPasswords(RestClient restClient) throws IOException {
RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, reservedPassword));
RequestOptions options = optionsBuilder.build();
for (String username : Arrays.asList(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
for (String username : Arrays.asList(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME, APMSystemUser.NAME)) {
Request request = new Request("PUT", "/_xpack/security/user/" + username + "/_password");
request.setJsonEntity("{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}");
request.setOptions(options);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
import org.elasticsearch.xpack.core.security.authc.support.Hasher;
import org.elasticsearch.xpack.core.security.user.APMSystemUser;
import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.core.security.user.ElasticUser;
import org.elasticsearch.xpack.core.security.user.KibanaUser;
Expand Down Expand Up @@ -81,7 +82,8 @@ void doExecute(Action<Response> action, Request request, ActionListener<Response
public void testPasswordUpsertWhenSetEnabledOnReservedUser() throws Exception {
final NativeUsersStore nativeUsersStore = startNativeUsersStore();

final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
BeatsSystemUser.NAME, APMSystemUser.NAME);

final PlainActionFuture<Void> future = new PlainActionFuture<>();
nativeUsersStore.setEnabled(user, true, WriteRequest.RefreshPolicy.IMMEDIATE, future);
Expand All @@ -99,7 +101,8 @@ public void testPasswordUpsertWhenSetEnabledOnReservedUser() throws Exception {
public void testBlankPasswordInIndexImpliesDefaultPassword() throws Exception {
final NativeUsersStore nativeUsersStore = startNativeUsersStore();

final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
final String user = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
BeatsSystemUser.NAME, APMSystemUser.NAME);
final Map<String, Object> values = new HashMap<>();
values.put(ENABLED_FIELD, Boolean.TRUE);
values.put(PASSWORD_FIELD, BLANK_PASSWORD);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@
import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse;
import org.elasticsearch.xpack.core.security.authc.support.Hasher;
import org.elasticsearch.xpack.core.security.client.SecurityClient;
import org.elasticsearch.xpack.core.security.user.APMSystemUser;
import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.core.security.user.ElasticUser;
import org.elasticsearch.xpack.core.security.user.KibanaUser;
import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
import org.junit.BeforeClass;

import java.util.Arrays;
import java.util.List;

import static java.util.Collections.singletonMap;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
Expand Down Expand Up @@ -49,7 +51,9 @@ public Settings nodeSettings(int nodeOrdinal) {
}

public void testAuthenticate() {
for (String username : Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
final List<String> usernames = Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
BeatsSystemUser.NAME, APMSystemUser.NAME);
for (String username : usernames) {
ClusterHealthResponse response = client()
.filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(username, getReservedPassword())))
.admin()
Expand All @@ -67,7 +71,9 @@ public void testAuthenticate() {
*/
public void testAuthenticateAfterEnablingUser() {
final SecurityClient c = securityClient();
for (String username : Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
final List<String> usernames = Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
BeatsSystemUser.NAME, APMSystemUser.NAME);
for (String username : usernames) {
c.prepareSetEnabled(username, true).get();
ClusterHealthResponse response = client()
.filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(username, getReservedPassword())))
Expand All @@ -81,7 +87,8 @@ public void testAuthenticateAfterEnablingUser() {
}

public void testChangingPassword() {
String username = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
String username = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
BeatsSystemUser.NAME, APMSystemUser.NAME);
final char[] newPassword = "supersecretvalue".toCharArray();

if (randomBoolean()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm;
import org.elasticsearch.xpack.core.security.authc.support.Hasher;
import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.core.security.user.APMSystemUser;
import org.elasticsearch.xpack.core.security.user.AnonymousUser;
import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
import org.elasticsearch.xpack.core.security.user.ElasticUser;
Expand Down Expand Up @@ -262,7 +263,8 @@ public void testGetUsers() {
PlainActionFuture<Collection<User>> userFuture = new PlainActionFuture<>();
reservedRealm.users(userFuture);
assertThat(userFuture.actionGet(),
containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true), new BeatsSystemUser(true)));
containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true),
new BeatsSystemUser(true), new APMSystemUser((true))));
}

public void testGetUsersDisabled() {
Expand Down Expand Up @@ -394,7 +396,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexExists
new AnonymousUser(Settings.EMPTY), securityIndex, threadPool);
PlainActionFuture<AuthenticationResult> listener = new PlainActionFuture<>();

final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME, APMSystemUser.NAME);
doAnswer((i) -> {
ActionListener callback = (ActionListener) i.getArguments()[1];
callback.onResponse(null);
Expand All @@ -416,14 +418,15 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexDoesNo
new AnonymousUser(Settings.EMPTY), securityIndex, threadPool);
PlainActionFuture<AuthenticationResult> listener = new PlainActionFuture<>();

final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME);
final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME, APMSystemUser.NAME);
reservedRealm.doAuthenticate(new UsernamePasswordToken(principal, mockSecureSettings.getString("bootstrap.password")), listener);
final AuthenticationResult result = listener.get();
assertThat(result.getStatus(), is(AuthenticationResult.Status.TERMINATE));
}

private User randomReservedUser(boolean enabled) {
return randomFrom(new ElasticUser(enabled), new KibanaUser(enabled), new LogstashSystemUser(enabled), new BeatsSystemUser(enabled));
return randomFrom(new ElasticUser(enabled), new KibanaUser(enabled), new LogstashSystemUser(enabled),
new BeatsSystemUser(enabled), new APMSystemUser(enabled));
}

/*
Expand Down Expand Up @@ -452,6 +455,11 @@ private void verifyVersionPredicate(String principal, Predicate<Version> version
assertThat(versionPredicate.test(Version.V_6_2_3), is(false));
assertThat(versionPredicate.test(Version.V_6_3_0), is(true));
break;
case APMSystemUser.NAME:
assertThat(versionPredicate.test(Version.V_5_6_9), is(false));
assertThat(versionPredicate.test(Version.V_6_4_0), is(false));
assertThat(versionPredicate.test(Version.V_6_5_0), is(true));
break;
default:
assertThat(versionPredicate.test(Version.V_6_3_0), is(true));
break;
Expand Down
Loading

0 comments on commit 1779d33

Please sign in to comment.