From 7d44d022c96f00ed6bcae9d92c714408a97402b9 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 7 Jul 2020 04:30:14 -0700 Subject: [PATCH] [APM] Adds 'Anomaly detection' settings page to create ML jobs per environment (#70560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds 'Anomaly detection' settings page along with require API endpoints to list and create the apm anomaly detection jobs per environment. Some test data is hardcoded while the the required changes in the ML plugin are in flight. * Converts the environment name to a compatible ML id string and persist in groups array. Also adds random token to the job ID to prevent collisions for job ids where diffferent environment names convert to the same string * - Improve job creation with latest updates for the `apm_transaction` ML module - Implements job list in settings by reading from `custom_settings.job_tags['service.environment']` - Add ML module method `createModuleItem` for job configuration - Don't allow user to type in duplicate environments * Update x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx Co-authored-by: Casper Hübertz * Update x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx Co-authored-by: Casper Hübertz * UX feedback, adds i18n, and handles failed state for ML jobs fetch. * - Moves get_all_environments from agent_configuration dir to common dir - makes the 'all' environment name ALL_OPTION_VALUE agent configuration-specific - replace field literals with constants * PR feedback * Adds support to create jobs for environment which are not defined. * Fixes description copy, rearranges settings links, and makes sure the 'Not defined' option is disabled if it already exists. * Only show "Not defined" in environment selector if there are actually documents without service.environment set * get the indexPatternName for the ML job from the set of user-definned indices * updated job_tags type definition Co-authored-by: Casper Hübertz Co-authored-by: Elastic Machine --- .../app/Main/route_config/index.tsx | 17 ++ .../app/Main/route_config/route_names.tsx | 1 + .../anomaly_detection/add_environments.tsx | 164 ++++++++++++++++++ .../Settings/anomaly_detection/create_jobs.ts | 64 +++++++ .../app/Settings/anomaly_detection/index.tsx | 68 ++++++++ .../Settings/anomaly_detection/jobs_list.tsx | 162 +++++++++++++++++ .../public/components/app/Settings/index.tsx | 23 ++- .../components/shared/ManagedTable/index.tsx | 11 +- .../create_anomaly_detection_jobs.ts | 123 +++++++++++++ .../get_anomaly_detection_jobs.ts | 60 +++++++ .../get_all_environments.test.ts.snap | 85 +++++++++ .../environments/get_all_environments.test.ts | 42 +++++ .../get_all_environments.ts | 13 +- .../apm/server/lib/helpers/setup_request.ts | 5 + .../__snapshots__/queries.test.ts.snap | 41 ----- .../get_environments/index.ts | 5 +- .../agent_configuration/queries.test.ts | 14 -- .../apm/server/routes/create_apm_api.ts | 12 +- .../routes/settings/anomaly_detection.ts | 55 ++++++ .../plugins/apm/typings/anomaly_detection.ts | 10 ++ .../types/anomaly_detection_jobs/job.ts | 3 + 21 files changed, 906 insertions(+), 72 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx create mode 100644 x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx create mode 100644 x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts create mode 100644 x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts create mode 100644 x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap create mode 100644 x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts rename x-pack/plugins/apm/server/lib/{settings/agent_configuration/get_environments => environments}/get_all_environments.ts (78%) create mode 100644 x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts create mode 100644 x-pack/plugins/apm/typings/anomaly_detection.ts diff --git a/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx b/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx index 1625fb4c1409ef..8379def2a7d9aa 100644 --- a/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx +++ b/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx @@ -23,6 +23,7 @@ import { resolveUrlParams } from '../../../../context/UrlParamsContext/resolveUr import { UNIDENTIFIED_SERVICE_NODES_LABEL } from '../../../../../common/i18n'; import { TraceLink } from '../../TraceLink'; import { CustomizeUI } from '../../Settings/CustomizeUI'; +import { AnomalyDetection } from '../../Settings/anomaly_detection'; import { EditAgentConfigurationRouteHandler, CreateAgentConfigurationRouteHandler, @@ -268,4 +269,20 @@ export const routes: BreadcrumbRoute[] = [ }), name: RouteName.RUM_OVERVIEW, }, + { + exact: true, + path: '/settings/anomaly-detection', + component: () => ( + + + + ), + breadcrumb: i18n.translate( + 'xpack.apm.breadcrumb.settings.anomalyDetection', + { + defaultMessage: 'Anomaly detection', + } + ), + name: RouteName.ANOMALY_DETECTION, + }, ]; diff --git a/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx b/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx index 4965aa9db87602..37d96e74d8ee6e 100644 --- a/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx +++ b/x-pack/plugins/apm/public/components/app/Main/route_config/route_names.tsx @@ -27,4 +27,5 @@ export enum RouteName { LINK_TO_TRACE = 'link_to_trace', CUSTOMIZE_UI = 'customize_ui', RUM_OVERVIEW = 'rum_overview', + ANOMALY_DETECTION = 'anomaly_detection', } diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx new file mode 100644 index 00000000000000..2da3c125631043 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/add_environments.tsx @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState } from 'react'; +import { + EuiPanel, + EuiTitle, + EuiText, + EuiSpacer, + EuiButton, + EuiButtonEmpty, + EuiComboBox, + EuiComboBoxOptionOption, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { useFetcher, FETCH_STATUS } from '../../../../hooks/useFetcher'; +import { useApmPluginContext } from '../../../../hooks/useApmPluginContext'; +import { createJobs } from './create_jobs'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../../../common/environment_filter_values'; + +interface Props { + currentEnvironments: string[]; + onCreateJobSuccess: () => void; + onCancel: () => void; +} +export const AddEnvironments = ({ + currentEnvironments, + onCreateJobSuccess, + onCancel, +}: Props) => { + const { toasts } = useApmPluginContext().core.notifications; + const { data = [], status } = useFetcher( + (callApmApi) => + callApmApi({ + pathname: `/api/apm/settings/anomaly-detection/environments`, + }), + [], + { preservePreviousData: false } + ); + + const environmentOptions = data.map((env) => ({ + label: env === ENVIRONMENT_NOT_DEFINED ? NOT_DEFINED_OPTION_LABEL : env, + value: env, + disabled: currentEnvironments.includes(env), + })); + + const [selectedOptions, setSelected] = useState< + Array> + >([]); + + const isLoading = + status === FETCH_STATUS.PENDING || status === FETCH_STATUS.LOADING; + return ( + + +

+ {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.titleText', + { + defaultMessage: 'Select environments', + } + )} +

+ + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.descriptionText', + { + defaultMessage: + 'Select the service environments that you want to enable anomaly detection in. Anomalies will surface for all services and transaction types within the selected environments.', + } + )} + + + + { + setSelected(nextSelectedOptions); + }} + onCreateOption={(searchValue) => { + if (currentEnvironments.includes(searchValue)) { + return; + } + const newOption = { + label: searchValue, + value: searchValue, + }; + setSelected([...selectedOptions, newOption]); + }} + isClearable={true} + /> + + + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.cancelButtonText', + { + defaultMessage: 'Cancel', + } + )} + + + + { + const selectedEnvironments = selectedOptions.map( + ({ value }) => value as string + ); + const success = await createJobs({ + environments: selectedEnvironments, + toasts, + }); + if (success) { + onCreateJobSuccess(); + } + }} + > + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.addEnvironments.createJobsButtonText', + { + defaultMessage: 'Create Jobs', + } + )} + + + + + + ); +}; + +const NOT_DEFINED_OPTION_LABEL = i18n.translate( + 'xpack.apm.filter.environment.notDefinedLabel', + { + defaultMessage: 'Not defined', + } +); diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts new file mode 100644 index 00000000000000..614632a5a3b092 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/create_jobs.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { NotificationsStart } from 'kibana/public'; +import { callApmApi } from '../../../../services/rest/createCallApmApi'; + +export async function createJobs({ + environments, + toasts, +}: { + environments: string[]; + toasts: NotificationsStart['toasts']; +}) { + try { + await callApmApi({ + pathname: '/api/apm/settings/anomaly-detection/jobs', + method: 'POST', + params: { + body: { environments }, + }, + }); + + toasts.addSuccess({ + title: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.succeeded.title', + { defaultMessage: 'Anomaly detection jobs created' } + ), + text: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.succeeded.text', + { + defaultMessage: + 'Anomaly detection jobs successfully created for APM service environments [{environments}]. It will take some time for machine learning to start analyzing traffic for anomalies.', + values: { environments: environments.join(', ') }, + } + ), + }); + return true; + } catch (error) { + toasts.addDanger({ + title: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.failed.title', + { + defaultMessage: 'Anomaly detection jobs could not be created', + } + ), + text: i18n.translate( + 'xpack.apm.anomalyDetection.createJobs.failed.text', + { + defaultMessage: + 'Something went wrong when creating one ore more anomaly detection jobs for APM service environments [{environments}]. Error: "{errorMessage}"', + values: { + environments: environments.join(', '), + errorMessage: error.message, + }, + } + ), + }); + return false; + } +} diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx new file mode 100644 index 00000000000000..0b720242237014 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState } from 'react'; +import { EuiTitle, EuiSpacer, EuiText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { JobsList } from './jobs_list'; +import { AddEnvironments } from './add_environments'; +import { useFetcher, FETCH_STATUS } from '../../../../hooks/useFetcher'; + +export const AnomalyDetection = () => { + const [viewAddEnvironments, setViewAddEnvironments] = useState(false); + + const { refetch, data = [], status } = useFetcher( + (callApmApi) => + callApmApi({ pathname: `/api/apm/settings/anomaly-detection` }), + [], + { preservePreviousData: false } + ); + + const isLoading = + status === FETCH_STATUS.PENDING || status === FETCH_STATUS.LOADING; + const hasFetchFailure = status === FETCH_STATUS.FAILURE; + + return ( + <> + +

+ {i18n.translate('xpack.apm.settings.anomalyDetection.titleText', { + defaultMessage: 'Anomaly detection', + })} +

+ + + + {i18n.translate('xpack.apm.settings.anomalyDetection.descriptionText', { + defaultMessage: + 'The Machine Learning anomaly detection integration enables application health status indicators in the Service map by identifying transaction duration anomalies.', + })} + + + {viewAddEnvironments ? ( + environment)} + onCreateJobSuccess={() => { + refetch(); + setViewAddEnvironments(false); + }} + onCancel={() => { + setViewAddEnvironments(false); + }} + /> + ) : ( + { + setViewAddEnvironments(true); + }} + /> + )} + + ); +}; diff --git a/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx new file mode 100644 index 00000000000000..30b4805011f03d --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { + EuiPanel, + EuiTitle, + EuiText, + EuiSpacer, + EuiButton, + EuiFlexGroup, + EuiFlexItem, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { ITableColumn, ManagedTable } from '../../../shared/ManagedTable'; +import { LoadingStatePrompt } from '../../../shared/LoadingStatePrompt'; +import { AnomalyDetectionJobByEnv } from '../../../../../typings/anomaly_detection'; +import { MLJobLink } from '../../../shared/Links/MachineLearningLinks/MLJobLink'; +import { MLLink } from '../../../shared/Links/MachineLearningLinks/MLLink'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../../../common/environment_filter_values'; + +const columns: Array> = [ + { + field: 'environment', + name: i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.environmentColumnLabel', + { defaultMessage: 'Environment' } + ), + render: (environment: string) => { + if (environment === ENVIRONMENT_NOT_DEFINED) { + return i18n.translate('xpack.apm.filter.environment.notDefinedLabel', { + defaultMessage: 'Not defined', + }); + } + return environment; + }, + }, + { + field: 'job_id', + align: 'right', + name: i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.actionColumnLabel', + { defaultMessage: 'Action' } + ), + render: (jobId: string) => ( + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.mlJobLinkText', + { + defaultMessage: 'View job in ML', + } + )} + + ), + }, +]; + +interface Props { + isLoading: boolean; + hasFetchFailure: boolean; + onAddEnvironments: () => void; + anomalyDetectionJobsByEnv: AnomalyDetectionJobByEnv[]; +} +export const JobsList = ({ + isLoading, + hasFetchFailure, + onAddEnvironments, + anomalyDetectionJobsByEnv, +}: Props) => { + return ( + + + + +

+ {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.environments', + { + defaultMessage: 'Environments', + } + )} +

+ + + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.addEnvironments', + { + defaultMessage: 'Add environments', + } + )} + + + + + + + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.mlDescriptionText.mlJobsLinkText', + { + defaultMessage: 'Machine Learning', + } + )} + + ), + }} + /> + + + + ) : hasFetchFailure ? ( + + ) : ( + + ) + } + columns={columns} + items={isLoading || hasFetchFailure ? [] : anomalyDetectionJobsByEnv} + /> + + + ); +}; + +function EmptyStatePrompt() { + return ( + <> + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.emptyListText', + { + defaultMessage: 'No anomaly detection jobs.', + } + )} + + ); +} + +function FailureStatePrompt() { + return ( + <> + {i18n.translate( + 'xpack.apm.settings.anomalyDetection.jobList.failedFetchText', + { + defaultMessage: 'Unabled to fetch anomaly detection jobs.', + } + )} + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/Settings/index.tsx b/x-pack/plugins/apm/public/components/app/Settings/index.tsx index 578a7db1958d42..6d8571bf577674 100644 --- a/x-pack/plugins/apm/public/components/app/Settings/index.tsx +++ b/x-pack/plugins/apm/public/components/app/Settings/index.tsx @@ -49,12 +49,15 @@ export const Settings: React.FC = (props) => { ), }, { - name: i18n.translate('xpack.apm.settings.indices', { - defaultMessage: 'Indices', - }), - id: '2', - href: getAPMHref('/settings/apm-indices', search), - isSelected: pathname === '/settings/apm-indices', + name: i18n.translate( + 'xpack.apm.settings.anomalyDetection', + { + defaultMessage: 'Anomaly detection', + } + ), + id: '4', + href: getAPMHref('/settings/anomaly-detection', search), + isSelected: pathname === '/settings/anomaly-detection', }, { name: i18n.translate('xpack.apm.settings.customizeApp', { @@ -64,6 +67,14 @@ export const Settings: React.FC = (props) => { href: getAPMHref('/settings/customize-ui', search), isSelected: pathname === '/settings/customize-ui', }, + { + name: i18n.translate('xpack.apm.settings.indices', { + defaultMessage: 'Indices', + }), + id: '2', + href: getAPMHref('/settings/apm-indices', search), + isSelected: pathname === '/settings/apm-indices', + }, ], }, ]} diff --git a/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx b/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx index 3dbb1b2faac020..50d46844f0adb7 100644 --- a/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/ManagedTable/index.tsx @@ -33,6 +33,7 @@ interface Props { hidePerPageOptions?: boolean; noItemsMessage?: React.ReactNode; sortItems?: boolean; + pagination?: boolean; } function UnoptimizedManagedTable(props: Props) { @@ -46,6 +47,7 @@ function UnoptimizedManagedTable(props: Props) { hidePerPageOptions = true, noItemsMessage, sortItems = true, + pagination = true, } = props; const { @@ -93,23 +95,26 @@ function UnoptimizedManagedTable(props: Props) { [] ); - const pagination = useMemo(() => { + const paginationProps = useMemo(() => { + if (!pagination) { + return; + } return { hidePerPageOptions, totalItemCount: items.length, pageIndex: page, pageSize, }; - }, [hidePerPageOptions, items, page, pageSize]); + }, [hidePerPageOptions, items, page, pageSize, pagination]); return ( >} // EuiBasicTableColumn is stricter than ITableColumn - pagination={pagination} sorting={sort} onChange={onTableChange} + {...(paginationProps ? { pagination: paginationProps } : {})} /> ); } diff --git a/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts new file mode 100644 index 00000000000000..406097805775d5 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'kibana/server'; +import uuid from 'uuid/v4'; +import { PromiseReturnType } from '../../../../observability/typings/common'; +import { Setup } from '../helpers/setup_request'; +import { + SERVICE_ENVIRONMENT, + TRANSACTION_DURATION, + PROCESSOR_EVENT, +} from '../../../common/elasticsearch_fieldnames'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; + +const ML_MODULE_ID_APM_TRANSACTION = 'apm_transaction'; +export const ML_GROUP_NAME_APM = 'apm'; + +export type CreateAnomalyDetectionJobsAPIResponse = PromiseReturnType< + typeof createAnomalyDetectionJobs +>; +export async function createAnomalyDetectionJobs( + setup: Setup, + environments: string[], + logger: Logger +) { + const { ml, indices } = setup; + if (!ml) { + logger.warn('Anomaly detection plugin is not available.'); + return []; + } + const mlCapabilities = await ml.mlSystem.mlCapabilities(); + if (!mlCapabilities.mlFeatureEnabledInSpace) { + logger.warn('Anomaly detection feature is not enabled for the space.'); + return []; + } + if (!mlCapabilities.isPlatinumOrTrialLicense) { + logger.warn( + 'Unable to create anomaly detection jobs due to insufficient license.' + ); + return []; + } + logger.info( + `Creating ML anomaly detection jobs for environments: [${environments}].` + ); + + const indexPatternName = indices['apm_oss.transactionIndices']; + const responses = await Promise.all( + environments.map((environment) => + createAnomalyDetectionJob({ ml, environment, indexPatternName }) + ) + ); + const jobResponses = responses.flatMap((response) => response.jobs); + const failedJobs = jobResponses.filter(({ success }) => !success); + + if (failedJobs.length > 0) { + const failedJobIds = failedJobs.map(({ id }) => id).join(', '); + logger.error( + `Failed to create anomaly detection ML jobs for: [${failedJobIds}]:` + ); + failedJobs.forEach(({ error }) => logger.error(JSON.stringify(error))); + throw new Error( + `Failed to create anomaly detection ML jobs for: [${failedJobIds}].` + ); + } + + return jobResponses; +} + +async function createAnomalyDetectionJob({ + ml, + environment, + indexPatternName = 'apm-*-transaction-*', +}: { + ml: Required['ml']; + environment: string; + indexPatternName?: string | undefined; +}) { + const convertedEnvironmentName = convertToMLIdentifier(environment); + const randomToken = uuid().substr(-4); + + return ml.modules.setup({ + moduleId: ML_MODULE_ID_APM_TRANSACTION, + prefix: `${ML_GROUP_NAME_APM}-${convertedEnvironmentName}-${randomToken}-`, + groups: [ML_GROUP_NAME_APM, convertedEnvironmentName], + indexPatternName, + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + { exists: { field: TRANSACTION_DURATION } }, + environment === ENVIRONMENT_NOT_DEFINED + ? ENVIRONMENT_NOT_DEFINED_FILTER + : { term: { [SERVICE_ENVIRONMENT]: environment } }, + ], + }, + }, + startDatafeed: true, + jobOverrides: [ + { + custom_settings: { + job_tags: { environment }, + }, + }, + ], + }); +} + +const ENVIRONMENT_NOT_DEFINED_FILTER = { + bool: { + must_not: { + exists: { + field: SERVICE_ENVIRONMENT, + }, + }, + }, +}; + +export function convertToMLIdentifier(value: string) { + return value.replace(/\s+/g, '_').toLowerCase(); +} diff --git a/x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts b/x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts new file mode 100644 index 00000000000000..252c87e9263db3 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'kibana/server'; +import { PromiseReturnType } from '../../../../observability/typings/common'; +import { Setup } from '../helpers/setup_request'; +import { AnomalyDetectionJobByEnv } from '../../../typings/anomaly_detection'; +import { ML_GROUP_NAME_APM } from './create_anomaly_detection_jobs'; + +export type AnomalyDetectionJobsAPIResponse = PromiseReturnType< + typeof getAnomalyDetectionJobs +>; +export async function getAnomalyDetectionJobs( + setup: Setup, + logger: Logger +): Promise { + const { ml } = setup; + if (!ml) { + return []; + } + try { + const mlCapabilities = await ml.mlSystem.mlCapabilities(); + if ( + !( + mlCapabilities.mlFeatureEnabledInSpace && + mlCapabilities.isPlatinumOrTrialLicense + ) + ) { + logger.warn( + 'Anomaly detection integration is not availble for this user.' + ); + return []; + } + } catch (error) { + logger.warn('Unable to get ML capabilities.'); + logger.error(error); + return []; + } + try { + const { jobs } = await ml.anomalyDetectors.jobs(ML_GROUP_NAME_APM); + return jobs + .map((job) => { + const environment = job.custom_settings?.job_tags?.environment ?? ''; + return { + job_id: job.job_id, + environment, + }; + }) + .filter((job) => job.environment); + } catch (error) { + if (error.statusCode !== 404) { + logger.warn('Unable to get APM ML jobs.'); + logger.error(error); + } + return []; + } +} diff --git a/x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap b/x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap new file mode 100644 index 00000000000000..b943102b39de82 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/environments/__snapshots__/get_all_environments.test.ts.snap @@ -0,0 +1,85 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`getAllEnvironments fetches all environments 1`] = ` +Object { + "body": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + "missing": undefined, + "size": 100, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "terms": Object { + "processor.event": Array [ + "transaction", + "error", + "metric", + ], + }, + }, + Object { + "term": Object { + "service.name": "test", + }, + }, + ], + }, + }, + "size": 0, + }, + "index": Array [ + "myIndex", + "myIndex", + "myIndex", + ], +} +`; + +exports[`getAllEnvironments fetches all environments with includeMissing 1`] = ` +Object { + "body": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + "missing": "ENVIRONMENT_NOT_DEFINED", + "size": 100, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "terms": Object { + "processor.event": Array [ + "transaction", + "error", + "metric", + ], + }, + }, + Object { + "term": Object { + "service.name": "test", + }, + }, + ], + }, + }, + "size": 0, + }, + "index": Array [ + "myIndex", + "myIndex", + "myIndex", + ], +} +`; diff --git a/x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts b/x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts new file mode 100644 index 00000000000000..25fc1776947446 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/environments/get_all_environments.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getAllEnvironments } from './get_all_environments'; +import { + SearchParamsMock, + inspectSearchParams, +} from '../../../public/utils/testHelpers'; + +describe('getAllEnvironments', () => { + let mock: SearchParamsMock; + + afterEach(() => { + mock.teardown(); + }); + + it('fetches all environments', async () => { + mock = await inspectSearchParams((setup) => + getAllEnvironments({ + serviceName: 'test', + setup, + }) + ); + + expect(mock.params).toMatchSnapshot(); + }); + + it('fetches all environments with includeMissing', async () => { + mock = await inspectSearchParams((setup) => + getAllEnvironments({ + serviceName: 'test', + setup, + includeMissing: true, + }) + ); + + expect(mock.params).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_all_environments.ts b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts similarity index 78% rename from x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_all_environments.ts rename to x-pack/plugins/apm/server/lib/environments/get_all_environments.ts index 88a528f12b41c9..9b17033a1f2a5e 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_all_environments.ts +++ b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts @@ -4,20 +4,22 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Setup } from '../../../helpers/setup_request'; +import { Setup } from '../helpers/setup_request'; import { PROCESSOR_EVENT, SERVICE_NAME, SERVICE_ENVIRONMENT, -} from '../../../../../common/elasticsearch_fieldnames'; -import { ALL_OPTION_VALUE } from '../../../../../common/agent_configuration/all_option'; +} from '../../../common/elasticsearch_fieldnames'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; export async function getAllEnvironments({ serviceName, setup, + includeMissing = false, }: { - serviceName: string | undefined; + serviceName?: string; setup: Setup; + includeMissing?: boolean; }) { const { client, indices } = setup; @@ -49,6 +51,7 @@ export async function getAllEnvironments({ terms: { field: SERVICE_ENVIRONMENT, size: 100, + missing: includeMissing ? ENVIRONMENT_NOT_DEFINED : undefined, }, }, }, @@ -60,5 +63,5 @@ export async function getAllEnvironments({ resp.aggregations?.environments.buckets.map( (bucket) => bucket.key as string ) || []; - return [ALL_OPTION_VALUE, ...environments]; + return environments; } diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts index 14c9378d991928..af073076a812a7 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts @@ -116,6 +116,11 @@ function getMlSetup(context: APMRequestHandlerContext, request: KibanaRequest) { return { mlSystem: ml.mlSystemProvider(mlClient, request), anomalyDetectors: ml.anomalyDetectorsProvider(mlClient, request), + modules: ml.modulesProvider( + mlClient, + request, + context.core.savedObjects.client + ), mlClient, }; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap index db34b4d5d20b5b..24a1840bc0ab87 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/__snapshots__/queries.test.ts.snap @@ -84,47 +84,6 @@ Object { } `; -exports[`agent configuration queries getAllEnvironments fetches all environments 1`] = ` -Object { - "body": Object { - "aggs": Object { - "environments": Object { - "terms": Object { - "field": "service.environment", - "size": 100, - }, - }, - }, - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "terms": Object { - "processor.event": Array [ - "transaction", - "error", - "metric", - ], - }, - }, - Object { - "term": Object { - "service.name": "foo", - }, - }, - ], - }, - }, - "size": 0, - }, - "index": Array [ - "myIndex", - "myIndex", - "myIndex", - ], -} -`; - exports[`agent configuration queries getExistingEnvironmentsForService fetches unavailable environments 1`] = ` Object { "body": Object { diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts index d10e06d1df632e..630249052be0b9 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/index.ts @@ -4,10 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getAllEnvironments } from './get_all_environments'; +import { getAllEnvironments } from '../../../environments/get_all_environments'; import { Setup } from '../../../helpers/setup_request'; import { PromiseReturnType } from '../../../../../../observability/typings/common'; import { getExistingEnvironmentsForService } from './get_existing_environments_for_service'; +import { ALL_OPTION_VALUE } from '../../../../../common/agent_configuration/all_option'; export type AgentConfigurationEnvironmentsAPIResponse = PromiseReturnType< typeof getEnvironments @@ -25,7 +26,7 @@ export async function getEnvironments({ getExistingEnvironmentsForService({ serviceName, setup }), ]); - return allEnvironments.map((environment) => { + return [ALL_OPTION_VALUE, ...allEnvironments].map((environment) => { return { name: environment, alreadyConfigured: existingEnvironments.includes(environment), diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts index 515376f8bb18be..5fe9d19ffc8605 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/queries.test.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getAllEnvironments } from './get_environments/get_all_environments'; import { getExistingEnvironmentsForService } from './get_environments/get_existing_environments_for_service'; import { getServiceNames } from './get_service_names'; import { listConfigurations } from './list_configurations'; @@ -22,19 +21,6 @@ describe('agent configuration queries', () => { mock.teardown(); }); - describe('getAllEnvironments', () => { - it('fetches all environments', async () => { - mock = await inspectSearchParams((setup) => - getAllEnvironments({ - serviceName: 'foo', - setup, - }) - ); - - expect(mock.params).toMatchSnapshot(); - }); - }); - describe('getExistingEnvironmentsForService', () => { it('fetches unavailable environments', async () => { mock = await inspectSearchParams((setup) => diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index ed1c045616a27c..c314debcd80493 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -81,6 +81,11 @@ import { observabilityDashboardHasDataRoute, observabilityDashboardDataRoute, } from './observability_dashboard'; +import { + anomalyDetectionJobsRoute, + createAnomalyDetectionJobsRoute, + anomalyDetectionEnvironmentsRoute, +} from './settings/anomaly_detection'; const createApmApi = () => { const api = createApi() @@ -170,7 +175,12 @@ const createApmApi = () => { // Observability dashboard .add(observabilityDashboardHasDataRoute) - .add(observabilityDashboardDataRoute); + .add(observabilityDashboardDataRoute) + + // Anomaly detection + .add(anomalyDetectionJobsRoute) + .add(createAnomalyDetectionJobsRoute) + .add(anomalyDetectionEnvironmentsRoute); return api; }; diff --git a/x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts b/x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts new file mode 100644 index 00000000000000..67eca0da946d0a --- /dev/null +++ b/x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { createRoute } from '../create_route'; +import { getAnomalyDetectionJobs } from '../../lib/anomaly_detection/get_anomaly_detection_jobs'; +import { createAnomalyDetectionJobs } from '../../lib/anomaly_detection/create_anomaly_detection_jobs'; +import { setupRequest } from '../../lib/helpers/setup_request'; +import { getAllEnvironments } from '../../lib/environments/get_all_environments'; + +// get ML anomaly detection jobs for each environment +export const anomalyDetectionJobsRoute = createRoute(() => ({ + method: 'GET', + path: '/api/apm/settings/anomaly-detection', + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + return await getAnomalyDetectionJobs(setup, context.logger); + }, +})); + +// create new ML anomaly detection jobs for each given environment +export const createAnomalyDetectionJobsRoute = createRoute(() => ({ + method: 'POST', + path: '/api/apm/settings/anomaly-detection/jobs', + options: { + tags: ['access:apm', 'access:apm_write'], + }, + params: { + body: t.type({ + environments: t.array(t.string), + }), + }, + handler: async ({ context, request }) => { + const { environments } = context.params.body; + const setup = await setupRequest(context, request); + return await createAnomalyDetectionJobs( + setup, + environments, + context.logger + ); + }, +})); + +// get all available environments to create anomaly detection jobs for +export const anomalyDetectionEnvironmentsRoute = createRoute(() => ({ + method: 'GET', + path: '/api/apm/settings/anomaly-detection/environments', + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + return await getAllEnvironments({ setup, includeMissing: true }); + }, +})); diff --git a/x-pack/plugins/apm/typings/anomaly_detection.ts b/x-pack/plugins/apm/typings/anomaly_detection.ts new file mode 100644 index 00000000000000..30dc92c36dea42 --- /dev/null +++ b/x-pack/plugins/apm/typings/anomaly_detection.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface AnomalyDetectionJobByEnv { + environment: string; + job_id: string; +} diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts index 3dbdb8bf3c0024..e2c4f1bae1a108 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/job.ts @@ -13,6 +13,9 @@ export type BucketSpan = string; export interface CustomSettings { custom_urls?: UrlConfig[]; created_by?: CREATED_BY_LABEL; + job_tags?: { + [tag: string]: string; + }; } export interface Job {