From 8ffc08f2f7bf1a017d4d875e9de869ecd2339d0d Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Tue, 7 Apr 2020 13:44:34 -0400 Subject: [PATCH] [REVERT THE REVERT] [Monitoring] Cluster state watch to Kibana alerting (#62793) * Revert "Revert "[Monitoring] Cluster state watch to Kibana alerting (#61685)"" This reverts commit f1bd3bdacb263b2754c04000070fe0f79ab5ed72. * Fix i18n error * Fix test --- .../plugins/monitoring/common/constants.ts | 8 +- .../public/components/alerts/alerts.js | 44 +- .../public/components/alerts/status.test.tsx | 8 +- .../public/components/alerts/status.tsx | 2 +- .../cluster/overview/alerts_panel.js | 81 ++- .../monitoring/public/views/alerts/index.js | 30 +- x-pack/plugins/monitoring/common/constants.ts | 8 +- .../server/alerts/cluster_state.test.ts | 186 ++++++ .../monitoring/server/alerts/cluster_state.ts | 134 ++++ .../plugins/monitoring/server/alerts/enums.ts | 16 + .../server/alerts/license_expiration.test.ts | 572 +++++------------- .../server/alerts/license_expiration.ts | 127 ++-- .../monitoring/server/alerts/types.d.ts | 62 +- .../lib/alerts/cluster_state.lib.test.ts | 70 +++ .../server/lib/alerts/cluster_state.lib.ts | 88 +++ .../lib/alerts/fetch_cluster_state.test.ts | 39 ++ .../server/lib/alerts/fetch_cluster_state.ts | 53 ++ .../server/lib/alerts/fetch_clusters.test.ts | 46 +- .../server/lib/alerts/fetch_clusters.ts | 41 +- .../server/lib/alerts/fetch_licenses.test.ts | 67 +- .../server/lib/alerts/fetch_licenses.ts | 16 +- .../server/lib/alerts/fetch_status.test.ts | 122 ++++ .../server/lib/alerts/fetch_status.ts | 100 ++- .../lib/alerts/get_prepared_alert.test.ts | 163 +++++ .../server/lib/alerts/get_prepared_alert.ts | 87 +++ .../lib/alerts/license_expiration.lib.test.ts | 23 +- .../lib/alerts/license_expiration.lib.ts | 52 +- .../lib/cluster/get_clusters_from_request.js | 12 +- x-pack/plugins/monitoring/server/plugin.ts | 12 + .../server/routes/api/v1/alerts/alerts.js | 53 +- 30 files changed, 1565 insertions(+), 757 deletions(-) create mode 100644 x-pack/plugins/monitoring/server/alerts/cluster_state.test.ts create mode 100644 x-pack/plugins/monitoring/server/alerts/cluster_state.ts create mode 100644 x-pack/plugins/monitoring/server/alerts/enums.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.test.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.test.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/fetch_status.test.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.test.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.ts diff --git a/x-pack/legacy/plugins/monitoring/common/constants.ts b/x-pack/legacy/plugins/monitoring/common/constants.ts index 9a4030f3eb2147..3a4c7b71dcd03d 100644 --- a/x-pack/legacy/plugins/monitoring/common/constants.ts +++ b/x-pack/legacy/plugins/monitoring/common/constants.ts @@ -239,11 +239,15 @@ export const ALERT_TYPE_PREFIX = 'monitoring_'; * This is the alert type id for the license expiration alert */ export const ALERT_TYPE_LICENSE_EXPIRATION = `${ALERT_TYPE_PREFIX}alert_type_license_expiration`; +/** + * This is the alert type id for the cluster state alert + */ +export const ALERT_TYPE_CLUSTER_STATE = `${ALERT_TYPE_PREFIX}alert_type_cluster_state`; /** * A listing of all alert types */ -export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION]; +export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION, ALERT_TYPE_CLUSTER_STATE]; /** * Matches the id for the built-in in email action type @@ -254,7 +258,7 @@ export const ALERT_ACTION_TYPE_EMAIL = '.email'; /** * The number of alerts that have been migrated */ -export const NUMBER_OF_MIGRATED_ALERTS = 1; +export const NUMBER_OF_MIGRATED_ALERTS = 2; /** * The advanced settings config name for the email address diff --git a/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js b/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js index 11fcef73a4b976..95c1af5549198f 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js +++ b/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js @@ -6,10 +6,15 @@ import React from 'react'; import chrome from '../../np_imports/ui/chrome'; -import { capitalize } from 'lodash'; +import { capitalize, get } from 'lodash'; import { formatDateTimeLocal } from '../../../common/formatting'; import { formatTimestampToDuration } from '../../../common'; -import { CALCULATE_DURATION_SINCE, EUI_SORT_DESCENDING } from '../../../common/constants'; +import { + CALCULATE_DURATION_SINCE, + EUI_SORT_DESCENDING, + ALERT_TYPE_LICENSE_EXPIRATION, + ALERT_TYPE_CLUSTER_STATE, +} from '../../../common/constants'; import { mapSeverity } from './map_severity'; import { FormattedAlert } from 'plugins/monitoring/components/alerts/formatted_alert'; import { EuiMonitoringTable } from 'plugins/monitoring/components/table'; @@ -21,6 +26,8 @@ const linkToCategories = { 'elasticsearch/indices': 'Elasticsearch Indices', 'kibana/instances': 'Kibana Instances', 'logstash/instances': 'Logstash Nodes', + [ALERT_TYPE_LICENSE_EXPIRATION]: 'License expiration', + [ALERT_TYPE_CLUSTER_STATE]: 'Cluster state', }; const getColumns = (kbnUrl, scope, timezone) => [ { @@ -94,19 +101,22 @@ const getColumns = (kbnUrl, scope, timezone) => [ }), field: 'message', sortable: true, - render: (message, alert) => ( - { - scope.$evalAsync(() => { - kbnUrl.changePath(target); - }); - }} - /> - ), + render: (_message, alert) => { + const message = get(alert, 'message.text', get(alert, 'message', '')); + return ( + { + scope.$evalAsync(() => { + kbnUrl.changePath(target); + }); + }} + /> + ); + }, }, { name: i18n.translate('xpack.monitoring.alerts.categoryColumnTitle', { @@ -148,8 +158,8 @@ const getColumns = (kbnUrl, scope, timezone) => [ export const Alerts = ({ alerts, angular, sorting, pagination, onTableChange }) => { const alertsFlattened = alerts.map(alert => ({ ...alert, - status: alert.metadata.severity, - category: alert.metadata.link, + status: get(alert, 'metadata.severity', get(alert, 'severity', 0)), + category: get(alert, 'metadata.link', get(alert, 'type', null)), })); const injector = chrome.dangerouslyGetActiveInjector(); diff --git a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx index 258a5b68db372e..d3cf4b463a2cc5 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx +++ b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { shallow } from 'enzyme'; import { kfetch } from 'ui/kfetch'; import { AlertsStatus, AlertsStatusProps } from './status'; -import { ALERT_TYPE_PREFIX } from '../../../common/constants'; +import { ALERT_TYPES } from '../../../common/constants'; import { getSetupModeState } from '../../lib/setup_mode'; import { mockUseEffects } from '../../jest.helpers'; @@ -63,11 +63,7 @@ describe('Status', () => { it('should render a success message if all alerts have been migrated and in setup mode', async () => { (kfetch as jest.Mock).mockReturnValue({ - data: [ - { - alertTypeId: ALERT_TYPE_PREFIX, - }, - ], + data: ALERT_TYPES.map(type => ({ alertTypeId: type })), }); (getSetupModeState as jest.Mock).mockReturnValue({ diff --git a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx index 072a98b1234521..5f5329bf7fff83 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx +++ b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx @@ -142,7 +142,7 @@ export const AlertsStatus: React.FC = (props: AlertsStatusPro ); } - const allMigrated = kibanaAlerts.length === NUMBER_OF_MIGRATED_ALERTS; + const allMigrated = kibanaAlerts.length >= NUMBER_OF_MIGRATED_ALERTS; if (allMigrated) { if (setupModeEnabled) { return ( diff --git a/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js b/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js index 8455fb8cf3088c..d87ff98e79be00 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js +++ b/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js @@ -6,14 +6,12 @@ import React, { Fragment } from 'react'; import moment from 'moment-timezone'; -import chrome from '../../../np_imports/ui/chrome'; import { FormattedAlert } from 'plugins/monitoring/components/alerts/formatted_alert'; import { mapSeverity } from 'plugins/monitoring/components/alerts/map_severity'; import { formatTimestampToDuration } from '../../../../common/format_timestamp_to_duration'; import { CALCULATE_DURATION_SINCE, KIBANA_ALERTING_ENABLED, - ALERT_TYPE_LICENSE_EXPIRATION, CALCULATE_DURATION_UNTIL, } from '../../../../common/constants'; import { formatDateTimeLocal } from '../../../../common/formatting'; @@ -31,6 +29,37 @@ import { EuiLink, } from '@elastic/eui'; +function replaceTokens(alert) { + if (!alert.message.tokens) { + return alert.message.text; + } + + let text = alert.message.text; + + for (const token of alert.message.tokens) { + if (token.type === 'time') { + text = text.replace( + token.startToken, + token.isRelative + ? formatTimestampToDuration(alert.expirationTime, CALCULATE_DURATION_UNTIL) + : moment.tz(alert.expirationTime, moment.tz.guess()).format('LLL z') + ); + } else if (token.type === 'link') { + const linkPart = new RegExp(`${token.startToken}(.+?)${token.endToken}`).exec(text); + // TODO: we assume this is at the end, which works for now but will not always work + const nonLinkText = text.replace(linkPart[0], ''); + text = ( + + {nonLinkText} + {linkPart[1]} + + ); + } + } + + return text; +} + export function AlertsPanel({ alerts, changeUrl }) { const goToAlerts = () => changeUrl('/alerts'); @@ -58,9 +87,6 @@ export function AlertsPanel({ alerts, changeUrl }) { severityIcon.iconType = 'check'; } - const injector = chrome.dangerouslyGetActiveInjector(); - const timezone = injector.get('config').get('dateFormat:tz'); - return ( @@ -96,14 +122,7 @@ export function AlertsPanel({ alerts, changeUrl }) { const alertsList = KIBANA_ALERTING_ENABLED ? alerts.map((alert, idx) => { const callOutProps = mapSeverity(alert.severity); - let message = alert.message - // scan message prefix and replace relative times - // \w: Matches any alphanumeric character from the basic Latin alphabet, including the underscore. Equivalent to [A-Za-z0-9_]. - .replace( - '#relative', - formatTimestampToDuration(alert.expirationTime, CALCULATE_DURATION_UNTIL) - ) - .replace('#absolute', moment.tz(alert.expirationTime, moment.tz.guess()).format('LLL z')); + const message = replaceTokens(alert); if (!alert.isFiring) { callOutProps.title = i18n.translate( @@ -118,22 +137,30 @@ export function AlertsPanel({ alerts, changeUrl }) { ); callOutProps.color = 'success'; callOutProps.iconType = 'check'; - } else { - if (alert.type === ALERT_TYPE_LICENSE_EXPIRATION) { - message = ( - - {message} -   - Please update your license - - ); - } } return ( - -

{message}

- + + +

{message}

+ +

+ +

+ + + + ); }) : alerts.map((item, index) => ( diff --git a/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js b/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js index 7c065a78a8af98..62cc985887e9f5 100644 --- a/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js +++ b/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js @@ -18,25 +18,37 @@ import { Alerts } from '../../components/alerts'; import { MonitoringViewBaseEuiTableController } from '../base_eui_table_controller'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiPage, EuiPageBody, EuiPageContent, EuiSpacer, EuiLink } from '@elastic/eui'; -import { CODE_PATH_ALERTS } from '../../../common/constants'; +import { CODE_PATH_ALERTS, KIBANA_ALERTING_ENABLED } from '../../../common/constants'; function getPageData($injector) { const globalState = $injector.get('globalState'); const $http = $injector.get('$http'); const Private = $injector.get('Private'); - const url = `../api/monitoring/v1/clusters/${globalState.cluster_uuid}/legacy_alerts`; + const url = KIBANA_ALERTING_ENABLED + ? `../api/monitoring/v1/alert_status` + : `../api/monitoring/v1/clusters/${globalState.cluster_uuid}/legacy_alerts`; const timeBounds = timefilter.getBounds(); + const data = { + timeRange: { + min: timeBounds.min.toISOString(), + max: timeBounds.max.toISOString(), + }, + }; + + if (!KIBANA_ALERTING_ENABLED) { + data.ccs = globalState.ccs; + } return $http - .post(url, { - ccs: globalState.ccs, - timeRange: { - min: timeBounds.min.toISOString(), - max: timeBounds.max.toISOString(), - }, + .post(url, data) + .then(response => { + const result = get(response, 'data', []); + if (KIBANA_ALERTING_ENABLED) { + return result.alerts; + } + return result; }) - .then(response => get(response, 'data', [])) .catch(err => { const ajaxErrorHandlers = Private(ajaxErrorHandlersProvider); return ajaxErrorHandlers(err); diff --git a/x-pack/plugins/monitoring/common/constants.ts b/x-pack/plugins/monitoring/common/constants.ts index 9a4030f3eb2147..3a4c7b71dcd03d 100644 --- a/x-pack/plugins/monitoring/common/constants.ts +++ b/x-pack/plugins/monitoring/common/constants.ts @@ -239,11 +239,15 @@ export const ALERT_TYPE_PREFIX = 'monitoring_'; * This is the alert type id for the license expiration alert */ export const ALERT_TYPE_LICENSE_EXPIRATION = `${ALERT_TYPE_PREFIX}alert_type_license_expiration`; +/** + * This is the alert type id for the cluster state alert + */ +export const ALERT_TYPE_CLUSTER_STATE = `${ALERT_TYPE_PREFIX}alert_type_cluster_state`; /** * A listing of all alert types */ -export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION]; +export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION, ALERT_TYPE_CLUSTER_STATE]; /** * Matches the id for the built-in in email action type @@ -254,7 +258,7 @@ export const ALERT_ACTION_TYPE_EMAIL = '.email'; /** * The number of alerts that have been migrated */ -export const NUMBER_OF_MIGRATED_ALERTS = 1; +export const NUMBER_OF_MIGRATED_ALERTS = 2; /** * The advanced settings config name for the email address diff --git a/x-pack/plugins/monitoring/server/alerts/cluster_state.test.ts b/x-pack/plugins/monitoring/server/alerts/cluster_state.test.ts new file mode 100644 index 00000000000000..6a9ca884373471 --- /dev/null +++ b/x-pack/plugins/monitoring/server/alerts/cluster_state.test.ts @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Logger } from 'src/core/server'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { getClusterState } from './cluster_state'; +import { AlertServices } from '../../../alerting/server'; +import { ALERT_TYPE_CLUSTER_STATE } from '../../common/constants'; +import { AlertCommonParams, AlertCommonState, AlertClusterStatePerClusterState } from './types'; +import { getPreparedAlert } from '../lib/alerts/get_prepared_alert'; +import { executeActions } from '../lib/alerts/cluster_state.lib'; +import { AlertClusterStateState } from './enums'; + +jest.mock('../lib/alerts/cluster_state.lib', () => ({ + executeActions: jest.fn(), + getUiMessage: jest.fn(), +})); + +jest.mock('../lib/alerts/get_prepared_alert', () => ({ + getPreparedAlert: jest.fn(() => { + return { + emailAddress: 'foo@foo.com', + }; + }), +})); + +interface MockServices { + callCluster: jest.Mock; + alertInstanceFactory: jest.Mock; + savedObjectsClient: jest.Mock; +} + +describe('getClusterState', () => { + const services: MockServices | AlertServices = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + savedObjectsClient: savedObjectsClientMock.create(), + }; + + const params: AlertCommonParams = { + dateFormat: 'YYYY', + timezone: 'UTC', + }; + + const emailAddress = 'foo@foo.com'; + const clusterUuid = 'kdksdfj434'; + const clusterName = 'monitoring_test'; + const cluster = { clusterUuid, clusterName }; + + async function setupAlert( + previousState: AlertClusterStateState, + newState: AlertClusterStateState + ): Promise { + const logger: Logger = { + warn: jest.fn(), + log: jest.fn(), + debug: jest.fn(), + trace: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + info: jest.fn(), + get: jest.fn(), + }; + const getLogger = (): Logger => logger; + const ccrEnabled = false; + (getPreparedAlert as jest.Mock).mockImplementation(() => ({ + emailAddress, + data: [ + { + state: newState, + clusterUuid, + }, + ], + clusters: [cluster], + })); + + const alert = getClusterState(null as any, null as any, getLogger, ccrEnabled); + const state: AlertCommonState = { + [clusterUuid]: { + state: previousState, + ui: { + isFiring: false, + severity: 0, + message: null, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }, + } as AlertClusterStatePerClusterState, + }; + + return (await alert.executor({ services, params, state } as any)) as AlertCommonState; + } + + afterEach(() => { + (executeActions as jest.Mock).mockClear(); + }); + + it('should configure the alert properly', () => { + const alert = getClusterState(null as any, null as any, jest.fn(), false); + expect(alert.id).toBe(ALERT_TYPE_CLUSTER_STATE); + expect(alert.actionGroups).toEqual([{ id: 'default', name: 'Default' }]); + }); + + it('should alert if green -> yellow', async () => { + const result = await setupAlert(AlertClusterStateState.Green, AlertClusterStateState.Yellow); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Yellow, + emailAddress + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Yellow); + expect(clusterResult.ui.isFiring).toBe(true); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should alert if yellow -> green', async () => { + const result = await setupAlert(AlertClusterStateState.Yellow, AlertClusterStateState.Green); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Green, + emailAddress, + true + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Green); + expect(clusterResult.ui.resolvedMS).toBeGreaterThan(0); + }); + + it('should alert if green -> red', async () => { + const result = await setupAlert(AlertClusterStateState.Green, AlertClusterStateState.Red); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Red, + emailAddress + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Red); + expect(clusterResult.ui.isFiring).toBe(true); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should alert if red -> green', async () => { + const result = await setupAlert(AlertClusterStateState.Red, AlertClusterStateState.Green); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Green, + emailAddress, + true + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Green); + expect(clusterResult.ui.resolvedMS).toBeGreaterThan(0); + }); + + it('should not alert if red -> yellow', async () => { + const result = await setupAlert(AlertClusterStateState.Red, AlertClusterStateState.Yellow); + expect(executeActions).not.toHaveBeenCalled(); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Red); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should not alert if yellow -> red', async () => { + const result = await setupAlert(AlertClusterStateState.Yellow, AlertClusterStateState.Red); + expect(executeActions).not.toHaveBeenCalled(); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Yellow); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should not alert if green -> green', async () => { + const result = await setupAlert(AlertClusterStateState.Green, AlertClusterStateState.Green); + expect(executeActions).not.toHaveBeenCalled(); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Green); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); +}); diff --git a/x-pack/plugins/monitoring/server/alerts/cluster_state.ts b/x-pack/plugins/monitoring/server/alerts/cluster_state.ts new file mode 100644 index 00000000000000..9a5805b8af7ced --- /dev/null +++ b/x-pack/plugins/monitoring/server/alerts/cluster_state.ts @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import moment from 'moment-timezone'; +import { i18n } from '@kbn/i18n'; +import { Logger, ICustomClusterClient, UiSettingsServiceStart } from 'src/core/server'; +import { ALERT_TYPE_CLUSTER_STATE } from '../../common/constants'; +import { AlertType } from '../../../alerting/server'; +import { executeActions, getUiMessage } from '../lib/alerts/cluster_state.lib'; +import { + AlertCommonExecutorOptions, + AlertCommonState, + AlertClusterStatePerClusterState, + AlertCommonCluster, +} from './types'; +import { AlertClusterStateState } from './enums'; +import { getPreparedAlert } from '../lib/alerts/get_prepared_alert'; +import { fetchClusterState } from '../lib/alerts/fetch_cluster_state'; + +export const getClusterState = ( + getUiSettingsService: () => Promise, + monitoringCluster: ICustomClusterClient, + getLogger: (...scopes: string[]) => Logger, + ccsEnabled: boolean +): AlertType => { + const logger = getLogger(ALERT_TYPE_CLUSTER_STATE); + return { + id: ALERT_TYPE_CLUSTER_STATE, + name: 'Monitoring Alert - Cluster Status', + actionGroups: [ + { + id: 'default', + name: i18n.translate('xpack.monitoring.alerts.clusterState.actionGroups.default', { + defaultMessage: 'Default', + }), + }, + ], + defaultActionGroupId: 'default', + async executor({ + services, + params, + state, + }: AlertCommonExecutorOptions): Promise { + logger.debug( + `Firing alert with params: ${JSON.stringify(params)} and state: ${JSON.stringify(state)}` + ); + + const preparedAlert = await getPreparedAlert( + ALERT_TYPE_CLUSTER_STATE, + getUiSettingsService, + monitoringCluster, + logger, + ccsEnabled, + services, + fetchClusterState + ); + + if (!preparedAlert) { + return state; + } + + const { emailAddress, data: states, clusters } = preparedAlert; + + const result: AlertCommonState = { ...state }; + const defaultAlertState: AlertClusterStatePerClusterState = { + state: AlertClusterStateState.Green, + ui: { + isFiring: false, + message: null, + severity: 0, + resolvedMS: 0, + triggeredMS: 0, + lastCheckedMS: 0, + }, + }; + + for (const clusterState of states) { + const alertState: AlertClusterStatePerClusterState = + (state[clusterState.clusterUuid] as AlertClusterStatePerClusterState) || + defaultAlertState; + const cluster = clusters.find( + (c: AlertCommonCluster) => c.clusterUuid === clusterState.clusterUuid + ); + if (!cluster) { + logger.warn(`Unable to find cluster for clusterUuid='${clusterState.clusterUuid}'`); + continue; + } + const isNonGreen = clusterState.state !== AlertClusterStateState.Green; + const severity = clusterState.state === AlertClusterStateState.Red ? 2100 : 1100; + + const ui = alertState.ui; + let triggered = ui.triggeredMS; + let resolved = ui.resolvedMS; + let message = ui.message || {}; + let lastState = alertState.state; + const instance = services.alertInstanceFactory(ALERT_TYPE_CLUSTER_STATE); + + if (isNonGreen) { + if (lastState === AlertClusterStateState.Green) { + logger.debug(`Cluster state changed from green to ${clusterState.state}`); + executeActions(instance, cluster, clusterState.state, emailAddress); + lastState = clusterState.state; + triggered = moment().valueOf(); + } + message = getUiMessage(clusterState.state); + resolved = 0; + } else if (!isNonGreen && lastState !== AlertClusterStateState.Green) { + logger.debug(`Cluster state changed from ${lastState} to green`); + executeActions(instance, cluster, clusterState.state, emailAddress, true); + lastState = clusterState.state; + message = getUiMessage(clusterState.state, true); + resolved = moment().valueOf(); + } + + result[clusterState.clusterUuid] = { + state: lastState, + ui: { + message, + isFiring: isNonGreen, + severity, + resolvedMS: resolved, + triggeredMS: triggered, + lastCheckedMS: moment().valueOf(), + }, + } as AlertClusterStatePerClusterState; + } + + return result; + }, + }; +}; diff --git a/x-pack/plugins/monitoring/server/alerts/enums.ts b/x-pack/plugins/monitoring/server/alerts/enums.ts new file mode 100644 index 00000000000000..ccff588743af1b --- /dev/null +++ b/x-pack/plugins/monitoring/server/alerts/enums.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export enum AlertClusterStateState { + Green = 'green', + Red = 'red', + Yellow = 'yellow', +} + +export enum AlertCommonPerClusterMessageTokenType { + Time = 'time', + Link = 'link', +} diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts index 0773af6e7f070d..92047e300bc1ff 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts @@ -6,42 +6,31 @@ import moment from 'moment-timezone'; import { getLicenseExpiration } from './license_expiration'; -import { - ALERT_TYPE_LICENSE_EXPIRATION, - MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS, -} from '../../common/constants'; +import { ALERT_TYPE_LICENSE_EXPIRATION } from '../../common/constants'; import { Logger } from 'src/core/server'; -import { AlertServices, AlertInstance } from '../../../alerting/server'; +import { AlertServices } from '../../../alerting/server'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { - AlertState, - AlertClusterState, - AlertParams, - LicenseExpirationAlertExecutorOptions, + AlertCommonParams, + AlertCommonState, + AlertLicensePerClusterState, + AlertLicense, } from './types'; -import { SavedObject, SavedObjectAttributes } from 'src/core/server'; -import { SavedObjectsClientContract } from 'src/core/server'; - -function fillLicense(license: any, clusterUuid?: string) { - return { - hits: { - hits: [ - { - _source: { - license, - cluster_uuid: clusterUuid, - }, - }, - ], - }, - }; -} - -const clusterUuid = 'a4545jhjb'; -const params: AlertParams = { - dateFormat: 'YYYY', - timezone: 'UTC', -}; +import { executeActions } from '../lib/alerts/license_expiration.lib'; +import { PreparedAlert, getPreparedAlert } from '../lib/alerts/get_prepared_alert'; + +jest.mock('../lib/alerts/license_expiration.lib', () => ({ + executeActions: jest.fn(), + getUiMessage: jest.fn(), +})); + +jest.mock('../lib/alerts/get_prepared_alert', () => ({ + getPreparedAlert: jest.fn(() => { + return { + emailAddress: 'foo@foo.com', + }; + }), +})); interface MockServices { callCluster: jest.Mock; @@ -49,428 +38,169 @@ interface MockServices { savedObjectsClient: jest.Mock; } -const alertExecutorOptions: LicenseExpirationAlertExecutorOptions = { - alertId: '', - startedAt: new Date(), - services: { - callCluster: (path: string, opts: any) => new Promise(resolve => resolve()), - alertInstanceFactory: (id: string) => new AlertInstance(), - savedObjectsClient: {} as jest.Mocked, - }, - params: {}, - state: {}, - spaceId: '', - name: '', - tags: [], - previousStartedAt: null, - createdBy: null, - updatedBy: null, -}; - describe('getLicenseExpiration', () => { - const emailAddress = 'foo@foo.com'; - const getUiSettingsService: any = () => ({ - asScopedToClient: (): any => ({ - get: () => new Promise(resolve => resolve(emailAddress)), - }), - }); - const monitoringCluster: any = null; - const logger: Logger = { - warn: jest.fn(), - log: jest.fn(), - debug: jest.fn(), - trace: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - info: jest.fn(), - get: jest.fn(), + const services: MockServices | AlertServices = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + savedObjectsClient: savedObjectsClientMock.create(), }; - const getLogger = (): Logger => logger; - const ccrEnabled = false; - afterEach(() => { - (logger.warn as jest.Mock).mockClear(); - }); - - it('should have the right id and actionGroups', () => { - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - expect(alert.id).toBe(ALERT_TYPE_LICENSE_EXPIRATION); - expect(alert.actionGroups).toEqual([{ id: 'default', name: 'Default' }]); - }); + const params: AlertCommonParams = { + dateFormat: 'YYYY', + timezone: 'UTC', + }; - it('should return the state if no license is provided', async () => { - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); + const emailAddress = 'foo@foo.com'; + const clusterUuid = 'kdksdfj434'; + const clusterName = 'monitoring_test'; + const dateFormat = 'YYYY-MM-DD'; + const cluster = { clusterUuid, clusterName }; + const defaultUiState = { + isFiring: false, + severity: 0, + message: null, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }; - const services: MockServices | AlertServices = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - savedObjectsClient: savedObjectsClientMock.create(), + async function setupAlert( + license: AlertLicense | null, + expiredCheckDateMS: number, + preparedAlertResponse: PreparedAlert | null | undefined = undefined + ): Promise { + const logger: Logger = { + warn: jest.fn(), + log: jest.fn(), + debug: jest.fn(), + trace: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + info: jest.fn(), + get: jest.fn(), }; - const state = { foo: 1 }; - - const result = await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - }); - - expect(result).toEqual(state); - }); + const getLogger = (): Logger => logger; + const ccrEnabled = false; + (getPreparedAlert as jest.Mock).mockImplementation(() => { + if (preparedAlertResponse !== undefined) { + return preparedAlertResponse; + } - it('should log a warning if no email is provided', async () => { - const customGetUiSettingsService: any = () => ({ - asScopedToClient: () => ({ - get: () => null, - }), + return { + emailAddress, + data: [license], + clusters: [cluster], + dateFormat, + }; }); - const alert = getLicenseExpiration( - customGetUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense({ - status: 'good', - type: 'basic', - expiry_date_in_millis: moment() - .add(7, 'days') - .valueOf(), - }) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory: jest.fn(), - savedObjectsClient: savedObjectsClientMock.create(), + const alert = getLicenseExpiration(null as any, null as any, getLogger, ccrEnabled); + const state: AlertCommonState = { + [clusterUuid]: { + expiredCheckDateMS, + ui: { ...defaultUiState }, + } as AlertLicensePerClusterState, }; - const state = {}; + return (await alert.executor({ services, params, state } as any)) as AlertCommonState; + } - await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - }); - - expect((logger.warn as jest.Mock).mock.calls.length).toBe(1); - expect(logger.warn).toHaveBeenCalledWith( - `Unable to send email for ${ALERT_TYPE_LICENSE_EXPIRATION} because there is no email configured.` - ); + afterEach(() => { + (executeActions as jest.Mock).mockClear(); + (getPreparedAlert as jest.Mock).mockClear(); }); - it('should fire actions if going to expire', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); + it('should have the right id and actionGroups', () => { + const alert = getLicenseExpiration(null as any, null as any, jest.fn(), false); + expect(alert.id).toBe(ALERT_TYPE_LICENSE_EXPIRATION); + expect(alert.actionGroups).toEqual([{ id: 'default', name: 'Default' }]); + }); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); + it('should return the state if no license is provided', async () => { + const result = await setupAlert(null, 0, null); + expect(result[clusterUuid].ui).toEqual(defaultUiState); + }); - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'gold', - expiry_date_in_millis: moment() - .add(7, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, + it('should fire actions if going to expire', async () => { + const expiryDateMS = moment() + .add(7, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'gold', + expiryDateMS, + clusterUuid, }; - - const state = {}; - - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; - + const result = await setupAlert(license, 0); + const newState = result[clusterUuid] as AlertLicensePerClusterState; expect(newState.expiredCheckDateMS > 0).toBe(true); - expect(scheduleActions.mock.calls.length).toBe(1); - expect(scheduleActions.mock.calls[0][1].subject).toBe( - 'NEW X-Pack Monitoring: License Expiration' + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + moment.utc(expiryDateMS), + dateFormat, + emailAddress ); - expect(scheduleActions.mock.calls[0][1].to).toBe(emailAddress); }); it('should fire actions if the user fixed their license', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'gold', - expiry_date_in_millis: moment() - .add(120, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, - }; - - const state: AlertState = { - [clusterUuid]: { - expiredCheckDateMS: moment() - .subtract(1, 'day') - .valueOf(), - ui: { isFiring: true, severity: 0, message: null, resolvedMS: 0, expirationTime: 0 }, - }, + const expiryDateMS = moment() + .add(365, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'gold', + expiryDateMS, + clusterUuid, }; - - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; + const result = await setupAlert(license, 100); + const newState = result[clusterUuid] as AlertLicensePerClusterState; expect(newState.expiredCheckDateMS).toBe(0); - expect(scheduleActions.mock.calls.length).toBe(1); - expect(scheduleActions.mock.calls[0][1].subject).toBe( - 'RESOLVED X-Pack Monitoring: License Expiration' + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + moment.utc(expiryDateMS), + dateFormat, + emailAddress, + true ); - expect(scheduleActions.mock.calls[0][1].to).toBe(emailAddress); }); it('should not fire actions for trial license that expire in more than 14 days', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'trial', - expiry_date_in_millis: moment() - .add(15, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, + const expiryDateMS = moment() + .add(20, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'trial', + expiryDateMS, + clusterUuid, }; - - const state = {}; - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; - expect(newState.expiredCheckDateMS).toBe(undefined); - expect(scheduleActions).not.toHaveBeenCalled(); + const result = await setupAlert(license, 0); + const newState = result[clusterUuid] as AlertLicensePerClusterState; + expect(newState.expiredCheckDateMS).toBe(0); + expect(executeActions).not.toHaveBeenCalled(); }); it('should fire actions for trial license that in 14 days or less', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'trial', - expiry_date_in_millis: moment() - .add(13, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, + const expiryDateMS = moment() + .add(7, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'trial', + expiryDateMS, + clusterUuid, }; - - const state = {}; - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; + const result = await setupAlert(license, 0); + const newState = result[clusterUuid] as AlertLicensePerClusterState; expect(newState.expiredCheckDateMS > 0).toBe(true); - expect(scheduleActions.mock.calls.length).toBe(1); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + moment.utc(expiryDateMS), + dateFormat, + emailAddress + ); }); }); diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration.ts index 93397ff3641ae1..2e5356150086b4 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration.ts @@ -5,24 +5,20 @@ */ import moment from 'moment-timezone'; -import { get } from 'lodash'; import { Logger, ICustomClusterClient, UiSettingsServiceStart } from 'src/core/server'; import { i18n } from '@kbn/i18n'; -import { ALERT_TYPE_LICENSE_EXPIRATION, INDEX_PATTERN_ELASTICSEARCH } from '../../common/constants'; +import { ALERT_TYPE_LICENSE_EXPIRATION } from '../../common/constants'; import { AlertType } from '../../../../plugins/alerting/server'; import { fetchLicenses } from '../lib/alerts/fetch_licenses'; -import { fetchDefaultEmailAddress } from '../lib/alerts/fetch_default_email_address'; -import { fetchClusters } from '../lib/alerts/fetch_clusters'; -import { fetchAvailableCcs } from '../lib/alerts/fetch_available_ccs'; import { - AlertLicense, - AlertState, - AlertClusterState, - AlertClusterUiState, - LicenseExpirationAlertExecutorOptions, + AlertCommonState, + AlertLicensePerClusterState, + AlertCommonExecutorOptions, + AlertCommonCluster, + AlertLicensePerClusterUiState, } from './types'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { executeActions, getUiMessage } from '../lib/alerts/license_expiration.lib'; +import { getPreparedAlert } from '../lib/alerts/get_prepared_alert'; const EXPIRES_DAYS = [60, 30, 14, 7]; @@ -32,14 +28,6 @@ export const getLicenseExpiration = ( getLogger: (...scopes: string[]) => Logger, ccsEnabled: boolean ): AlertType => { - async function getCallCluster(services: any): Promise { - if (!monitoringCluster) { - return services.callCluster; - } - - return monitoringCluster.callAsInternalUser; - } - const logger = getLogger(ALERT_TYPE_LICENSE_EXPIRATION); return { id: ALERT_TYPE_LICENSE_EXPIRATION, @@ -53,54 +41,50 @@ export const getLicenseExpiration = ( }, ], defaultActionGroupId: 'default', - async executor({ - services, - params, - state, - }: LicenseExpirationAlertExecutorOptions): Promise { + async executor({ services, params, state }: AlertCommonExecutorOptions): Promise { logger.debug( `Firing alert with params: ${JSON.stringify(params)} and state: ${JSON.stringify(state)}` ); - const callCluster = await getCallCluster(services); - - // Support CCS use cases by querying to find available remote clusters - // and then adding those to the index pattern we are searching against - let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH; - if (ccsEnabled) { - const availableCcs = await fetchAvailableCcs(callCluster); - if (availableCcs.length > 0) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } - } - - const clusters = await fetchClusters(callCluster, esIndexPattern); + const preparedAlert = await getPreparedAlert( + ALERT_TYPE_LICENSE_EXPIRATION, + getUiSettingsService, + monitoringCluster, + logger, + ccsEnabled, + services, + fetchLicenses + ); - // Fetch licensing information from cluster_stats documents - const licenses: AlertLicense[] = await fetchLicenses(callCluster, clusters, esIndexPattern); - if (licenses.length === 0) { - logger.warn(`No license found for ${ALERT_TYPE_LICENSE_EXPIRATION}.`); + if (!preparedAlert) { return state; } - const uiSettings = (await getUiSettingsService()).asScopedToClient( - services.savedObjectsClient - ); - const dateFormat: string = await uiSettings.get('dateFormat'); - const timezone: string = await uiSettings.get('dateFormat:tz'); - const emailAddress = await fetchDefaultEmailAddress(uiSettings); - if (!emailAddress) { - // TODO: we can do more here - logger.warn( - `Unable to send email for ${ALERT_TYPE_LICENSE_EXPIRATION} because there is no email configured.` - ); - return; - } + const { emailAddress, data: licenses, clusters, dateFormat } = preparedAlert; - const result: AlertState = { ...state }; + const result: AlertCommonState = { ...state }; + const defaultAlertState: AlertLicensePerClusterState = { + expiredCheckDateMS: 0, + ui: { + isFiring: false, + message: null, + severity: 0, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }, + }; for (const license of licenses) { - const licenseState: AlertClusterState = state[license.clusterUuid] || {}; + const alertState: AlertLicensePerClusterState = + (state[license.clusterUuid] as AlertLicensePerClusterState) || defaultAlertState; + const cluster = clusters.find( + (c: AlertCommonCluster) => c.clusterUuid === license.clusterUuid + ); + if (!cluster) { + logger.warn(`Unable to find cluster for clusterUuid='${license.clusterUuid}'`); + continue; + } const $expiry = moment.utc(license.expiryDateMS); let isExpired = false; let severity = 0; @@ -123,31 +107,26 @@ export const getLicenseExpiration = ( } } - const ui: AlertClusterUiState = get(licenseState, 'ui', { - isFiring: false, - message: null, - severity: 0, - resolvedMS: 0, - expirationTime: 0, - }); + const ui = alertState.ui; + let triggered = ui.triggeredMS; let resolved = ui.resolvedMS; let message = ui.message; - let expiredCheckDate = licenseState.expiredCheckDateMS; + let expiredCheckDate = alertState.expiredCheckDateMS; const instance = services.alertInstanceFactory(ALERT_TYPE_LICENSE_EXPIRATION); if (isExpired) { - if (!licenseState.expiredCheckDateMS) { + if (!alertState.expiredCheckDateMS) { logger.debug(`License will expire soon, sending email`); - executeActions(instance, license, $expiry, dateFormat, emailAddress); - expiredCheckDate = moment().valueOf(); + executeActions(instance, cluster, $expiry, dateFormat, emailAddress); + expiredCheckDate = triggered = moment().valueOf(); } - message = getUiMessage(license, timezone); + message = getUiMessage(); resolved = 0; - } else if (!isExpired && licenseState.expiredCheckDateMS) { + } else if (!isExpired && alertState.expiredCheckDateMS) { logger.debug(`License expiration has been resolved, sending email`); - executeActions(instance, license, $expiry, dateFormat, emailAddress, true); + executeActions(instance, cluster, $expiry, dateFormat, emailAddress, true); expiredCheckDate = 0; - message = getUiMessage(license, timezone, true); + message = getUiMessage(true); resolved = moment().valueOf(); } @@ -159,8 +138,10 @@ export const getLicenseExpiration = ( isFiring: expiredCheckDate > 0, severity, resolvedMS: resolved, - }, - }; + triggeredMS: triggered, + lastCheckedMS: moment().valueOf(), + } as AlertLicensePerClusterUiState, + } as AlertLicensePerClusterState; } return result; diff --git a/x-pack/plugins/monitoring/server/alerts/types.d.ts b/x-pack/plugins/monitoring/server/alerts/types.d.ts index ff47d6f2ad4dca..b689d008b51a78 100644 --- a/x-pack/plugins/monitoring/server/alerts/types.d.ts +++ b/x-pack/plugins/monitoring/server/alerts/types.d.ts @@ -5,41 +5,79 @@ */ import { Moment } from 'moment'; import { AlertExecutorOptions } from '../../../alerting/server'; +import { AlertClusterStateState, AlertCommonPerClusterMessageTokenType } from './enums'; export interface AlertLicense { status: string; type: string; expiryDateMS: number; clusterUuid: string; - clusterName: string; } -export interface AlertState { - [clusterUuid: string]: AlertClusterState; +export interface AlertClusterState { + state: AlertClusterStateState; + clusterUuid: string; +} + +export interface AlertCommonState { + [clusterUuid: string]: AlertCommonPerClusterState; } -export interface AlertClusterState { - expiredCheckDateMS: number | Moment; - ui: AlertClusterUiState; +export interface AlertCommonPerClusterState { + ui: AlertCommonPerClusterUiState; } -export interface AlertClusterUiState { +export interface AlertClusterStatePerClusterState extends AlertCommonPerClusterState { + state: AlertClusterStateState; +} + +export interface AlertLicensePerClusterState extends AlertCommonPerClusterState { + expiredCheckDateMS: number; +} + +export interface AlertCommonPerClusterUiState { isFiring: boolean; severity: number; - message: string | null; + message: AlertCommonPerClusterMessage | null; resolvedMS: number; + lastCheckedMS: number; + triggeredMS: number; +} + +export interface AlertCommonPerClusterMessage { + text: string; // Do this. #link this is a link #link + tokens?: AlertCommonPerClusterMessageToken[]; +} + +export interface AlertCommonPerClusterMessageToken { + startToken: string; + endToken?: string; + type: AlertCommonPerClusterMessageTokenType; +} + +export interface AlertCommonPerClusterMessageLinkToken extends AlertCommonPerClusterMessageToken { + url?: string; +} + +export interface AlertCommonPerClusterMessageTimeToken extends AlertCommonPerClusterMessageToken { + isRelative: boolean; + isAbsolute: boolean; +} + +export interface AlertLicensePerClusterUiState extends AlertCommonPerClusterUiState { expirationTime: number; } -export interface AlertCluster { +export interface AlertCommonCluster { clusterUuid: string; + clusterName: string; } -export interface LicenseExpirationAlertExecutorOptions extends AlertExecutorOptions { - state: AlertState; +export interface AlertCommonExecutorOptions extends AlertExecutorOptions { + state: AlertCommonState; } -export interface AlertParams { +export interface AlertCommonParams { dateFormat: string; timezone: string; } diff --git a/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.test.ts new file mode 100644 index 00000000000000..81e375734cc507 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { executeActions, getUiMessage } from './cluster_state.lib'; +import { AlertClusterStateState } from '../../alerts/enums'; +import { AlertCommonPerClusterMessageLinkToken } from '../../alerts/types'; + +describe('clusterState lib', () => { + describe('executeActions', () => { + const clusterName = 'clusterA'; + const instance: any = { scheduleActions: jest.fn() }; + const license: any = { clusterName }; + const status = AlertClusterStateState.Green; + const emailAddress = 'test@test.com'; + + beforeEach(() => { + instance.scheduleActions.mockClear(); + }); + + it('should schedule actions when firing', () => { + executeActions(instance, license, status, emailAddress, false); + expect(instance.scheduleActions).toHaveBeenCalledWith('default', { + subject: 'NEW X-Pack Monitoring: Cluster Status', + message: `Allocate missing replica shards for cluster '${clusterName}'`, + to: emailAddress, + }); + }); + + it('should have a different message for red state', () => { + executeActions(instance, license, AlertClusterStateState.Red, emailAddress, false); + expect(instance.scheduleActions).toHaveBeenCalledWith('default', { + subject: 'NEW X-Pack Monitoring: Cluster Status', + message: `Allocate missing primary and replica shards for cluster '${clusterName}'`, + to: emailAddress, + }); + }); + + it('should schedule actions when resolved', () => { + executeActions(instance, license, status, emailAddress, true); + expect(instance.scheduleActions).toHaveBeenCalledWith('default', { + subject: 'RESOLVED X-Pack Monitoring: Cluster Status', + message: `This cluster alert has been resolved: Allocate missing replica shards for cluster '${clusterName}'`, + to: emailAddress, + }); + }); + }); + + describe('getUiMessage', () => { + it('should return a message when firing', () => { + const message = getUiMessage(AlertClusterStateState.Red, false); + expect(message.text).toBe( + `Elasticsearch cluster status is red. #start_linkAllocate missing primary and replica shards#end_link` + ); + expect(message.tokens && message.tokens.length).toBe(1); + expect(message.tokens && message.tokens[0].startToken).toBe('#start_link'); + expect(message.tokens && message.tokens[0].endToken).toBe('#end_link'); + expect( + message.tokens && (message.tokens[0] as AlertCommonPerClusterMessageLinkToken).url + ).toBe('elasticsearch/indices'); + }); + + it('should return a message when resolved', () => { + const message = getUiMessage(AlertClusterStateState.Green, true); + expect(message.text).toBe(`Elasticsearch cluster status is green.`); + expect(message.tokens).not.toBeDefined(); + }); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.ts b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.ts new file mode 100644 index 00000000000000..ae66d603507ca6 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { i18n } from '@kbn/i18n'; +import { AlertInstance } from '../../../../alerting/server'; +import { + AlertCommonCluster, + AlertCommonPerClusterMessage, + AlertCommonPerClusterMessageLinkToken, +} from '../../alerts/types'; +import { AlertClusterStateState, AlertCommonPerClusterMessageTokenType } from '../../alerts/enums'; + +const RESOLVED_SUBJECT = i18n.translate('xpack.monitoring.alerts.clusterStatus.resolvedSubject', { + defaultMessage: 'RESOLVED X-Pack Monitoring: Cluster Status', +}); + +const NEW_SUBJECT = i18n.translate('xpack.monitoring.alerts.clusterStatus.newSubject', { + defaultMessage: 'NEW X-Pack Monitoring: Cluster Status', +}); + +const RED_STATUS_MESSAGE = i18n.translate('xpack.monitoring.alerts.clusterStatus.redMessage', { + defaultMessage: 'Allocate missing primary and replica shards', +}); + +const YELLOW_STATUS_MESSAGE = i18n.translate( + 'xpack.monitoring.alerts.clusterStatus.yellowMessage', + { + defaultMessage: 'Allocate missing replica shards', + } +); + +export function executeActions( + instance: AlertInstance, + cluster: AlertCommonCluster, + status: AlertClusterStateState, + emailAddress: string, + resolved: boolean = false +) { + const message = + status === AlertClusterStateState.Red ? RED_STATUS_MESSAGE : YELLOW_STATUS_MESSAGE; + if (resolved) { + instance.scheduleActions('default', { + subject: RESOLVED_SUBJECT, + message: `This cluster alert has been resolved: ${message} for cluster '${cluster.clusterName}'`, + to: emailAddress, + }); + } else { + instance.scheduleActions('default', { + subject: NEW_SUBJECT, + message: `${message} for cluster '${cluster.clusterName}'`, + to: emailAddress, + }); + } +} + +export function getUiMessage( + status: AlertClusterStateState, + resolved: boolean = false +): AlertCommonPerClusterMessage { + if (resolved) { + return { + text: i18n.translate('xpack.monitoring.alerts.clusterStatus.ui.resolvedMessage', { + defaultMessage: `Elasticsearch cluster status is green.`, + }), + }; + } + const message = + status === AlertClusterStateState.Red ? RED_STATUS_MESSAGE : YELLOW_STATUS_MESSAGE; + return { + text: i18n.translate('xpack.monitoring.alerts.clusterStatus.ui.firingMessage', { + defaultMessage: `Elasticsearch cluster status is {status}. #start_link{message}#end_link`, + values: { + status, + message, + }, + }), + tokens: [ + { + startToken: '#start_link', + endToken: '#end_link', + type: AlertCommonPerClusterMessageTokenType.Link, + url: 'elasticsearch/indices', + } as AlertCommonPerClusterMessageLinkToken, + ], + }; +} diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.test.ts new file mode 100644 index 00000000000000..642ae3c39a0275 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fetchClusterState } from './fetch_cluster_state'; + +describe('fetchClusterState', () => { + it('should return the cluster state', async () => { + const status = 'green'; + const clusterUuid = 'sdfdsaj34434'; + const callCluster = jest.fn(() => ({ + hits: { + hits: [ + { + _source: { + cluster_state: { + status, + }, + cluster_uuid: clusterUuid, + }, + }, + ], + }, + })); + + const clusters = [{ clusterUuid, clusterName: 'foo' }]; + const index = '.monitoring-es-*'; + + const state = await fetchClusterState(callCluster, clusters, index); + expect(state).toEqual([ + { + state: status, + clusterUuid, + }, + ]); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.ts new file mode 100644 index 00000000000000..66ea30d5f2e96a --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { get } from 'lodash'; +import { AlertCommonCluster, AlertClusterState } from '../../alerts/types'; + +export async function fetchClusterState( + callCluster: any, + clusters: AlertCommonCluster[], + index: string +): Promise { + const params = { + index, + filterPath: ['hits.hits._source.cluster_state.status', 'hits.hits._source.cluster_uuid'], + body: { + size: 1, + sort: [{ timestamp: { order: 'desc' } }], + query: { + bool: { + filter: [ + { + terms: { + cluster_uuid: clusters.map(cluster => cluster.clusterUuid), + }, + }, + { + term: { + type: 'cluster_stats', + }, + }, + { + range: { + timestamp: { + gte: 'now-2m', + }, + }, + }, + ], + }, + }, + }, + }; + + const response = await callCluster('search', params); + return get(response, 'hits.hits', []).map((hit: any) => { + return { + state: get(hit, '_source.cluster_state.status'), + clusterUuid: get(hit, '_source.cluster_uuid'), + }; + }); +} diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts index 78eb9773df15fb..7a9b61f37707ba 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts @@ -6,21 +6,51 @@ import { fetchClusters } from './fetch_clusters'; describe('fetchClusters', () => { + const clusterUuid = '1sdfds734'; + const clusterName = 'monitoring'; + it('return a list of clusters', async () => { const callCluster = jest.fn().mockImplementation(() => ({ - aggregations: { - clusters: { - buckets: [ - { - key: 'clusterA', + hits: { + hits: [ + { + _source: { + cluster_uuid: clusterUuid, + cluster_name: clusterName, + }, + }, + ], + }, + })); + const index = '.monitoring-es-*'; + const result = await fetchClusters(callCluster, index); + expect(result).toEqual([{ clusterUuid, clusterName }]); + }); + + it('return the metadata name if available', async () => { + const metadataName = 'custom-monitoring'; + const callCluster = jest.fn().mockImplementation(() => ({ + hits: { + hits: [ + { + _source: { + cluster_uuid: clusterUuid, + cluster_name: clusterName, + cluster_settings: { + cluster: { + metadata: { + display_name: metadataName, + }, + }, + }, }, - ], - }, + }, + ], }, })); const index = '.monitoring-es-*'; const result = await fetchClusters(callCluster, index); - expect(result).toEqual([{ clusterUuid: 'clusterA' }]); + expect(result).toEqual([{ clusterUuid, clusterName: metadataName }]); }); it('should limit the time period in the query', async () => { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts index 8ef7339618a2c0..d1513ac16fb158 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts @@ -4,18 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ import { get } from 'lodash'; -import { AlertCluster } from '../../alerts/types'; +import { AlertCommonCluster } from '../../alerts/types'; -interface AggregationResult { - key: string; -} - -export async function fetchClusters(callCluster: any, index: string): Promise { +export async function fetchClusters( + callCluster: any, + index: string +): Promise { const params = { index, - filterPath: 'aggregations.clusters.buckets', + filterPath: [ + 'hits.hits._source.cluster_settings.cluster.metadata.display_name', + 'hits.hits._source.cluster_uuid', + 'hits.hits._source.cluster_name', + ], body: { - size: 0, + size: 1000, query: { bool: { filter: [ @@ -34,19 +37,21 @@ export async function fetchClusters(callCluster: any, index: string): Promise ({ - clusterUuid: bucket.key, - })); + return get(response, 'hits.hits', []).map((hit: any) => { + const clusterName: string = + get(hit, '_source.cluster_settings.cluster.metadata.display_name') || + get(hit, '_source.cluster_name') || + get(hit, '_source.cluster_uuid'); + return { + clusterUuid: get(hit, '_source.cluster_uuid'), + clusterName, + }; + }); } diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts index dd6c074e68b1f9..9dcb4ffb82a5fc 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts @@ -6,28 +6,28 @@ import { fetchLicenses } from './fetch_licenses'; describe('fetchLicenses', () => { + const clusterName = 'MyCluster'; + const clusterUuid = 'clusterA'; + const license = { + status: 'active', + expiry_date_in_millis: 1579532493876, + type: 'basic', + }; + it('return a list of licenses', async () => { - const clusterName = 'MyCluster'; - const clusterUuid = 'clusterA'; - const license = { - status: 'active', - expiry_date_in_millis: 1579532493876, - type: 'basic', - }; const callCluster = jest.fn().mockImplementation(() => ({ hits: { hits: [ { _source: { license, - cluster_name: clusterName, cluster_uuid: clusterUuid, }, }, ], }, })); - const clusters = [{ clusterUuid }]; + const clusters = [{ clusterUuid, clusterName }]; const index = '.monitoring-es-*'; const result = await fetchLicenses(callCluster, clusters, index); expect(result).toEqual([ @@ -36,15 +36,13 @@ describe('fetchLicenses', () => { type: license.type, expiryDateMS: license.expiry_date_in_millis, clusterUuid, - clusterName, }, ]); }); it('should only search for the clusters provided', async () => { - const clusterUuid = 'clusterA'; const callCluster = jest.fn(); - const clusters = [{ clusterUuid }]; + const clusters = [{ clusterUuid, clusterName }]; const index = '.monitoring-es-*'; await fetchLicenses(callCluster, clusters, index); const params = callCluster.mock.calls[0][1]; @@ -52,54 +50,11 @@ describe('fetchLicenses', () => { }); it('should limit the time period in the query', async () => { - const clusterUuid = 'clusterA'; const callCluster = jest.fn(); - const clusters = [{ clusterUuid }]; + const clusters = [{ clusterUuid, clusterName }]; const index = '.monitoring-es-*'; await fetchLicenses(callCluster, clusters, index); const params = callCluster.mock.calls[0][1]; expect(params.body.query.bool.filter[2].range.timestamp.gte).toBe('now-2m'); }); - - it('should give priority to the metadata name', async () => { - const clusterName = 'MyCluster'; - const clusterUuid = 'clusterA'; - const license = { - status: 'active', - expiry_date_in_millis: 1579532493876, - type: 'basic', - }; - const callCluster = jest.fn().mockImplementation(() => ({ - hits: { - hits: [ - { - _source: { - license, - cluster_name: 'fakeName', - cluster_uuid: clusterUuid, - cluster_settings: { - cluster: { - metadata: { - display_name: clusterName, - }, - }, - }, - }, - }, - ], - }, - })); - const clusters = [{ clusterUuid }]; - const index = '.monitoring-es-*'; - const result = await fetchLicenses(callCluster, clusters, index); - expect(result).toEqual([ - { - status: license.status, - type: license.type, - expiryDateMS: license.expiry_date_in_millis, - clusterUuid, - clusterName, - }, - ]); - }); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts index 31a68e8aa9c3ee..5b05c907e796e8 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts @@ -4,21 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ import { get } from 'lodash'; -import { AlertLicense, AlertCluster } from '../../alerts/types'; +import { AlertLicense, AlertCommonCluster } from '../../alerts/types'; export async function fetchLicenses( callCluster: any, - clusters: AlertCluster[], + clusters: AlertCommonCluster[], index: string ): Promise { const params = { index, - filterPath: [ - 'hits.hits._source.license.*', - 'hits.hits._source.cluster_settings.cluster.metadata.display_name', - 'hits.hits._source.cluster_uuid', - 'hits.hits._source.cluster_name', - ], + filterPath: ['hits.hits._source.license.*', 'hits.hits._source.cluster_uuid'], body: { size: 1, sort: [{ timestamp: { order: 'desc' } }], @@ -50,17 +45,12 @@ export async function fetchLicenses( const response = await callCluster('search', params); return get(response, 'hits.hits', []).map((hit: any) => { - const clusterName: string = - get(hit, '_source.cluster_settings.cluster.metadata.display_name') || - get(hit, '_source.cluster_name') || - get(hit, '_source.cluster_uuid'); const rawLicense: any = get(hit, '_source.license', {}); const license: AlertLicense = { status: rawLicense.status, type: rawLicense.type, expiryDateMS: rawLicense.expiry_date_in_millis, clusterUuid: get(hit, '_source.cluster_uuid'), - clusterName, }; return license; }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.test.ts new file mode 100644 index 00000000000000..a3bcb61afacd61 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fetchStatus } from './fetch_status'; +import { AlertCommonPerClusterState } from '../../alerts/types'; + +describe('fetchStatus', () => { + const alertType = 'monitoringTest'; + const log = { warn: jest.fn() }; + const start = 0; + const end = 0; + const id = 1; + const defaultUiState = { + isFiring: false, + severity: 0, + message: null, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }; + const alertsClient = { + find: jest.fn(() => ({ + total: 1, + data: [ + { + id, + }, + ], + })), + getAlertState: jest.fn(() => ({ + alertTypeState: { + state: { + ui: defaultUiState, + } as AlertCommonPerClusterState, + }, + })), + }; + + afterEach(() => { + (alertsClient.find as jest.Mock).mockClear(); + (alertsClient.getAlertState as jest.Mock).mockClear(); + }); + + it('should fetch from the alerts client', async () => { + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status).toEqual([]); + }); + + it('should return alerts that are firing', async () => { + alertsClient.getAlertState = jest.fn(() => ({ + alertTypeState: { + state: { + ui: { + ...defaultUiState, + isFiring: true, + }, + } as AlertCommonPerClusterState, + }, + })); + + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status.length).toBe(1); + expect(status[0].type).toBe(alertType); + expect(status[0].isFiring).toBe(true); + }); + + it('should return alerts that have been resolved in the time period', async () => { + alertsClient.getAlertState = jest.fn(() => ({ + alertTypeState: { + state: { + ui: { + ...defaultUiState, + resolvedMS: 1500, + }, + } as AlertCommonPerClusterState, + }, + })); + + const customStart = 1000; + const customEnd = 2000; + + const status = await fetchStatus( + alertsClient as any, + [alertType], + customStart, + customEnd, + log as any + ); + expect(status.length).toBe(1); + expect(status[0].type).toBe(alertType); + expect(status[0].isFiring).toBe(false); + }); + + it('should pass in the right filter to the alerts client', async () => { + await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect((alertsClient.find as jest.Mock).mock.calls[0][0].options.filter).toBe( + `alert.attributes.alertTypeId:${alertType}` + ); + }); + + it('should return nothing if no alert state is found', async () => { + alertsClient.getAlertState = jest.fn(() => ({ + alertTypeState: null, + })) as any; + + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status).toEqual([]); + }); + + it('should return nothing if no alerts are found', async () => { + alertsClient.find = jest.fn(() => ({ + total: 0, + data: [], + })) as any; + + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status).toEqual([]); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts index 9f7c1d5a994d27..bf6ee965d3b2fb 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts @@ -4,81 +4,53 @@ * you may not use this file except in compliance with the Elastic License. */ import moment from 'moment'; -import { get } from 'lodash'; -import { AlertClusterState } from '../../alerts/types'; -import { ALERT_TYPES, LOGGING_TAG } from '../../../common/constants'; +import { Logger } from '../../../../../../src/core/server'; +import { AlertCommonPerClusterState } from '../../alerts/types'; +import { AlertsClient } from '../../../../alerting/server'; export async function fetchStatus( - callCluster: any, + alertsClient: AlertsClient, + alertTypes: string[], start: number, end: number, - clusterUuid: string, - server: any + log: Logger ): Promise { - // TODO: this shouldn't query task manager directly but rather - // use an api exposed by the alerting/actions plugin - // See https://github.com/elastic/kibana/issues/48442 const statuses = await Promise.all( - ALERT_TYPES.map( + alertTypes.map( type => new Promise(async (resolve, reject) => { - try { - const params = { - index: '.kibana_task_manager', - filterPath: ['hits.hits._source.task.state'], - body: { - size: 1, - sort: [{ updated_at: { order: 'desc' } }], - query: { - bool: { - filter: [ - { - term: { - 'task.taskType': `alerting:${type}`, - }, - }, - ], - }, - }, - }, - }; - - const response = await callCluster('search', params); - const state = get(response, 'hits.hits[0]._source.task.state', '{}'); - const clusterState: AlertClusterState = get( - JSON.parse(state), - `alertTypeState.${clusterUuid}`, - { - expiredCheckDateMS: 0, - ui: { - isFiring: false, - message: null, - severity: 0, - resolvedMS: 0, - expirationTime: 0, - }, - } - ); - const isInBetween = moment(clusterState.ui.resolvedMS).isBetween(start, end); - if (clusterState.ui.isFiring || isInBetween) { - return resolve({ - type, - ...clusterState.ui, - }); - } + // We need to get the id from the alertTypeId + const alerts = await alertsClient.find({ + options: { + filter: `alert.attributes.alertTypeId:${type}`, + }, + }); + if (alerts.total === 0) { return resolve(false); - } catch (err) { - const reason = get(err, 'body.error.type'); - if (reason === 'index_not_found_exception') { - server.log( - ['error', LOGGING_TAG], - `Unable to fetch alerts. Alerts depends on task manager, which has not been started yet.` - ); - } else { - server.log(['error', LOGGING_TAG], err.message); - } + } + + if (alerts.total !== 1) { + log.warn(`Found more than one alert for type ${type} which is unexpected.`); + } + + const id = alerts.data[0].id; + + // Now that we have the id, we can get the state + const states = await alertsClient.getAlertState({ id }); + if (!states || !states.alertTypeState) { + log.warn(`No alert states found for type ${type} which is unexpected.`); return resolve(false); } + + const state = Object.values(states.alertTypeState)[0] as AlertCommonPerClusterState; + const isInBetween = moment(state.ui.resolvedMS).isBetween(start, end); + if (state.ui.isFiring || isInBetween) { + return resolve({ + type, + ...state.ui, + }); + } + return resolve(false); }) ) ); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.test.ts new file mode 100644 index 00000000000000..1840a2026a7534 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getPreparedAlert } from './get_prepared_alert'; +import { fetchClusters } from './fetch_clusters'; +import { fetchDefaultEmailAddress } from './fetch_default_email_address'; + +jest.mock('./fetch_clusters', () => ({ + fetchClusters: jest.fn(), +})); + +jest.mock('./fetch_default_email_address', () => ({ + fetchDefaultEmailAddress: jest.fn(), +})); + +describe('getPreparedAlert', () => { + const uiSettings = { get: jest.fn() }; + const alertType = 'test'; + const getUiSettingsService = async () => ({ + asScopedToClient: () => uiSettings, + }); + const monitoringCluster = null; + const logger = { warn: jest.fn() }; + const ccsEnabled = false; + const services = { + callCluster: jest.fn(), + savedObjectsClient: null, + }; + const emailAddress = 'foo@foo.com'; + const data = [{ foo: 1 }]; + const dataFetcher = () => data; + const clusterName = 'MonitoringCluster'; + const clusterUuid = 'sdf34sdf'; + const clusters = [{ clusterName, clusterUuid }]; + + afterEach(() => { + (uiSettings.get as jest.Mock).mockClear(); + (services.callCluster as jest.Mock).mockClear(); + (fetchClusters as jest.Mock).mockClear(); + (fetchDefaultEmailAddress as jest.Mock).mockClear(); + }); + + beforeEach(() => { + (fetchClusters as jest.Mock).mockImplementation(() => clusters); + (fetchDefaultEmailAddress as jest.Mock).mockImplementation(() => emailAddress); + }); + + it('should return fields as expected', async () => { + (uiSettings.get as jest.Mock).mockImplementation(() => { + return emailAddress; + }); + + const alert = await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + ccsEnabled, + services as any, + dataFetcher as any + ); + + expect(alert && alert.emailAddress).toBe(emailAddress); + expect(alert && alert.data).toBe(data); + }); + + it('should add ccs if specified', async () => { + const ccsClusterName = 'remoteCluster'; + (services.callCluster as jest.Mock).mockImplementation(() => { + return { + [ccsClusterName]: { + connected: true, + }, + }; + }); + + await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + dataFetcher as any + ); + + expect((fetchClusters as jest.Mock).mock.calls[0][1].includes(ccsClusterName)).toBe(true); + }); + + it('should ignore ccs if no remote clusters are available', async () => { + const ccsClusterName = 'remoteCluster'; + (services.callCluster as jest.Mock).mockImplementation(() => { + return { + [ccsClusterName]: { + connected: false, + }, + }; + }); + + await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + dataFetcher as any + ); + + expect((fetchClusters as jest.Mock).mock.calls[0][1].includes(ccsClusterName)).toBe(false); + }); + + it('should pass in the clusters into the data fetcher', async () => { + const customDataFetcher = jest.fn(() => data); + + await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + customDataFetcher as any + ); + + expect((customDataFetcher as jest.Mock).mock.calls[0][1]).toBe(clusters); + }); + + it('should return nothing if the data fetcher returns nothing', async () => { + const customDataFetcher = jest.fn(() => []); + + const result = await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + customDataFetcher as any + ); + + expect(result).toBe(null); + }); + + it('should return nothing if there is no email address', async () => { + (fetchDefaultEmailAddress as jest.Mock).mockImplementation(() => null); + + const result = await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + dataFetcher as any + ); + + expect(result).toBe(null); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.ts b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.ts new file mode 100644 index 00000000000000..83a9e26e4c5890 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger, ICustomClusterClient, UiSettingsServiceStart } from 'kibana/server'; +import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; +import { AlertServices } from '../../../../alerting/server'; +import { AlertCommonCluster } from '../../alerts/types'; +import { INDEX_PATTERN_ELASTICSEARCH } from '../../../common/constants'; +import { fetchAvailableCcs } from './fetch_available_ccs'; +import { getCcsIndexPattern } from './get_ccs_index_pattern'; +import { fetchClusters } from './fetch_clusters'; +import { fetchDefaultEmailAddress } from './fetch_default_email_address'; + +export interface PreparedAlert { + emailAddress: string; + clusters: AlertCommonCluster[]; + data: any[]; + timezone: string; + dateFormat: string; +} + +async function getCallCluster( + monitoringCluster: ICustomClusterClient, + services: Pick +): Promise { + if (!monitoringCluster) { + return services.callCluster; + } + + return monitoringCluster.callAsInternalUser; +} + +export async function getPreparedAlert( + alertType: string, + getUiSettingsService: () => Promise, + monitoringCluster: ICustomClusterClient, + logger: Logger, + ccsEnabled: boolean, + services: Pick, + dataFetcher: ( + callCluster: CallCluster, + clusters: AlertCommonCluster[], + esIndexPattern: string + ) => Promise +): Promise { + const callCluster = await getCallCluster(monitoringCluster, services); + + // Support CCS use cases by querying to find available remote clusters + // and then adding those to the index pattern we are searching against + let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH; + if (ccsEnabled) { + const availableCcs = await fetchAvailableCcs(callCluster); + if (availableCcs.length > 0) { + esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); + } + } + + const clusters = await fetchClusters(callCluster, esIndexPattern); + + // Fetch the specific data + const data = await dataFetcher(callCluster, clusters, esIndexPattern); + if (data.length === 0) { + logger.warn(`No data found for ${alertType}.`); + return null; + } + + const uiSettings = (await getUiSettingsService()).asScopedToClient(services.savedObjectsClient); + const dateFormat: string = await uiSettings.get('dateFormat'); + const timezone: string = await uiSettings.get('dateFormat:tz'); + const emailAddress = await fetchDefaultEmailAddress(uiSettings); + if (!emailAddress) { + // TODO: we can do more here + logger.warn(`Unable to send email for ${alertType} because there is no email configured.`); + return null; + } + + return { + emailAddress, + data, + clusters, + dateFormat, + timezone, + }; +} diff --git a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts index 1a2eb1e44be843..b99208bdde2c86 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts @@ -39,17 +39,26 @@ describe('licenseExpiration lib', () => { }); describe('getUiMessage', () => { - const timezone = 'Europe/London'; - const license: any = { expiryDateMS: moment.tz('2020-01-20 08:00:00', timezone).utc() }; - it('should return a message when firing', () => { - const message = getUiMessage(license, timezone, false); - expect(message).toBe(`This cluster's license is going to expire in #relative at #absolute.`); + const message = getUiMessage(false); + expect(message.text).toBe( + `This cluster's license is going to expire in #relative at #absolute. #start_linkPlease update your license.#end_link` + ); + // LOL How do I avoid this in TS???? + if (!message.tokens) { + return expect(false).toBe(true); + } + expect(message.tokens.length).toBe(3); + expect(message.tokens[0].startToken).toBe('#relative'); + expect(message.tokens[1].startToken).toBe('#absolute'); + expect(message.tokens[2].startToken).toBe('#start_link'); + expect(message.tokens[2].endToken).toBe('#end_link'); }); it('should return a message when resolved', () => { - const message = getUiMessage(license, timezone, true); - expect(message).toBe(`This cluster's license is active.`); + const message = getUiMessage(true); + expect(message.text).toBe(`This cluster's license is active.`); + expect(message.tokens).not.toBeDefined(); }); }); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts index 41b68d69bbd25f..cfe9f02b9bd6ae 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts @@ -6,7 +6,13 @@ import { Moment } from 'moment-timezone'; import { i18n } from '@kbn/i18n'; import { AlertInstance } from '../../../../alerting/server'; -import { AlertLicense } from '../../alerts/types'; +import { + AlertCommonPerClusterMessageLinkToken, + AlertCommonPerClusterMessageTimeToken, + AlertCommonCluster, + AlertCommonPerClusterMessage, +} from '../../alerts/types'; +import { AlertCommonPerClusterMessageTokenType } from '../../alerts/enums'; const RESOLVED_SUBJECT = i18n.translate( 'xpack.monitoring.alerts.licenseExpiration.resolvedSubject', @@ -21,7 +27,7 @@ const NEW_SUBJECT = i18n.translate('xpack.monitoring.alerts.licenseExpiration.ne export function executeActions( instance: AlertInstance, - license: AlertLicense, + cluster: AlertCommonCluster, $expiry: Moment, dateFormat: string, emailAddress: string, @@ -31,14 +37,14 @@ export function executeActions( instance.scheduleActions('default', { subject: RESOLVED_SUBJECT, message: `This cluster alert has been resolved: Cluster '${ - license.clusterName + cluster.clusterName }' license was going to expire on ${$expiry.format(dateFormat)}.`, to: emailAddress, }); } else { instance.scheduleActions('default', { subject: NEW_SUBJECT, - message: `Cluster '${license.clusterName}' license is going to expire on ${$expiry.format( + message: `Cluster '${cluster.clusterName}' license is going to expire on ${$expiry.format( dateFormat )}. Please update your license.`, to: emailAddress, @@ -46,13 +52,37 @@ export function executeActions( } } -export function getUiMessage(license: AlertLicense, timezone: string, resolved: boolean = false) { +export function getUiMessage(resolved: boolean = false): AlertCommonPerClusterMessage { if (resolved) { - return i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.resolvedMessage', { - defaultMessage: `This cluster's license is active.`, - }); + return { + text: i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.resolvedMessage', { + defaultMessage: `This cluster's license is active.`, + }), + }; } - return i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.firingMessage', { - defaultMessage: `This cluster's license is going to expire in #relative at #absolute.`, - }); + return { + text: i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.firingMessage', { + defaultMessage: `This cluster's license is going to expire in #relative at #absolute. #start_linkPlease update your license.#end_link`, + }), + tokens: [ + { + startToken: '#relative', + type: AlertCommonPerClusterMessageTokenType.Time, + isRelative: true, + isAbsolute: false, + } as AlertCommonPerClusterMessageTimeToken, + { + startToken: '#absolute', + type: AlertCommonPerClusterMessageTokenType.Time, + isAbsolute: true, + isRelative: false, + } as AlertCommonPerClusterMessageTimeToken, + { + startToken: '#start_link', + endToken: '#end_link', + type: AlertCommonPerClusterMessageTokenType.Link, + url: 'license', + } as AlertCommonPerClusterMessageLinkToken, + ], + }; } diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js index c5091c36c3bbeb..1bddede52207b2 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js @@ -29,6 +29,7 @@ import { CODE_PATH_BEATS, CODE_PATH_APM, KIBANA_ALERTING_ENABLED, + ALERT_TYPES, } from '../../../common/constants'; import { getApmsForClusters } from '../apm/get_apms_for_clusters'; import { i18n } from '@kbn/i18n'; @@ -102,15 +103,8 @@ export async function getClustersFromRequest( if (isInCodePath(codePaths, [CODE_PATH_ALERTS])) { if (KIBANA_ALERTING_ENABLED) { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); - const callCluster = (...args) => callWithRequest(req, ...args); - cluster.alerts = await fetchStatus( - callCluster, - start, - end, - cluster.cluster_uuid, - req.server - ); + const alertsClient = req.getAlertsClient ? req.getAlertsClient() : null; + cluster.alerts = await fetchStatus(alertsClient, ALERT_TYPES, start, end, req.logger); } else { cluster.alerts = await alertsClusterSearch( req, diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index 24d8bcaa4397cf..784226dca66fee 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -47,6 +47,7 @@ import { PluginSetupContract as AlertingPluginSetupContract, } from '../../alerting/server'; import { getLicenseExpiration } from './alerts/license_expiration'; +import { getClusterState } from './alerts/cluster_state'; import { InfraPluginSetup } from '../../infra/server'; export interface LegacyAPI { @@ -154,6 +155,17 @@ export class Plugin { config.ui.ccs.enabled ) ); + plugins.alerting.registerType( + getClusterState( + async () => { + const coreStart = (await core.getStartServices())[0]; + return coreStart.uiSettings; + }, + cluster, + this.getLogger, + config.ui.ccs.enabled + ) + ); } // Initialize telemetry diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js b/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js index 56922bd8e87e21..d5a43d32f600a9 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js @@ -8,8 +8,12 @@ import { schema } from '@kbn/config-schema'; import { isFunction } from 'lodash'; import { ALERT_TYPE_LICENSE_EXPIRATION, + ALERT_TYPE_CLUSTER_STATE, MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS, + ALERT_TYPES, } from '../../../../../common/constants'; +import { handleError } from '../../../../lib/errors'; +import { fetchStatus } from '../../../../lib/alerts/fetch_status'; async function createAlerts(req, alertsClient, { selectedEmailActionId }) { const createdAlerts = []; @@ -17,7 +21,21 @@ async function createAlerts(req, alertsClient, { selectedEmailActionId }) { // Create alerts const ALERT_TYPES = { [ALERT_TYPE_LICENSE_EXPIRATION]: { - schedule: { interval: '10s' }, + schedule: { interval: '1m' }, + actions: [ + { + group: 'default', + id: selectedEmailActionId, + params: { + subject: '{{context.subject}}', + message: `{{context.message}}`, + to: ['{{context.to}}'], + }, + }, + ], + }, + [ALERT_TYPE_CLUSTER_STATE]: { + schedule: { interval: '1m' }, actions: [ { group: 'default', @@ -86,4 +104,37 @@ export function createKibanaAlertsRoute(server) { return { alerts, emailResponse }; }, }); + + server.route({ + method: 'POST', + path: '/api/monitoring/v1/alert_status', + config: { + validate: { + payload: schema.object({ + timeRange: schema.object({ + min: schema.string(), + max: schema.string(), + }), + }), + }, + }, + async handler(req, headers) { + const alertsClient = isFunction(req.getAlertsClient) ? req.getAlertsClient() : null; + if (!alertsClient) { + return headers.response().code(404); + } + + const start = req.payload.timeRange.min; + const end = req.payload.timeRange.max; + let alerts; + + try { + alerts = await fetchStatus(alertsClient, ALERT_TYPES, start, end, req.logger); + } catch (err) { + throw handleError(err, req); + } + + return { alerts }; + }, + }); }